[READ-ONLY] Mirror of https://github.com/FoxxMD/komodo-import. Import existing compose stacks into Komodo
foxxmd.github.io/komodo-import
compose
docker
import
komodo
toml
feat!: Re-architect stacks parsing to support parsed compose project data and simplify all stack generation
* Remove/rename directory ENVs to be clearer in description
* Introduce STACKS_FROM source of truth ENV
* Remove rigid monorepo mode, make git stack builder responsible for finding git up parent tree
* Use parsed compose project data to supplement stack building (name, compose files)
+450
-220
+11
-7
src/builders/stack/composeUtils.ts
+11
-7
src/builders/stack/composeUtils.ts
···
2
2
import { Container } from "../../common/dockerApi.js";
3
3
import { StackCandidate, StackCandidateCompose } from "../../common/infrastructure/atomic.js";
4
4
5
-
export const consolidateComposeStacks = (containers: Container[], parentLogger: Logger): StackCandidate[] => {
5
+
export const consolidateComposeStacks = (containers: Container[], parentLogger: Logger): StackCandidateCompose[] => {
6
6
const stacks: Record<string, StackCandidateCompose> = {};
7
7
const logger = childLogger(parentLogger, 'Compose Parser')
8
8
9
-
for(const c of containers) {
9
+
logger.verbose('Finding Compose projects for reference...');
10
+
11
+
for (const c of containers) {
10
12
const name = c.Labels['com.docker.compose.project'];
11
-
if(name !== undefined && stacks[name] === undefined) {
13
+
if (name !== undefined && stacks[name] === undefined) {
12
14
const candidate: StackCandidateCompose = {
13
15
discovered: 'compose',
14
16
projectName: name,
···
18
20
state: c.State
19
21
}
20
22
let ok = true;
21
-
for(const [k,v] of Object.entries(candidate)) {
22
-
if(v === undefined) {
23
+
for (const [k, v] of Object.entries(candidate)) {
24
+
if (v === undefined) {
23
25
logger.warn(`Cannot use ${name} compose project because ${k} label is missing`);
24
26
ok = false;
25
27
}
26
28
}
27
-
if(!ok) {
29
+
if (!ok) {
28
30
continue;
29
31
}
30
32
logger.verbose(`Found Project '${candidate.projectName}' at working dir '${candidate.workingDir}'`)
31
33
stacks[name] = candidate;
32
34
}
33
35
}
34
-
return Object.values(stacks);
36
+
const s = Object.values(stacks);
37
+
logger.verbose(`Found ${s.length} Compose projects`);
38
+
return s;
35
39
}
+11
-4
src/builders/stack/filesOnServer.ts
+11
-4
src/builders/stack/filesOnServer.ts
···
4
4
import { parse, ParsedPath, sep, join } from 'path';
5
5
import { TomlStack } from "../../common/infrastructure/tomlObjects.js";
6
6
import { readText } from "../../common/utils/io.js";
7
-
import { isDebugMode, removeUndefinedKeys } from "../../common/utils/utils.js";
7
+
import { isDebugMode, removeRootPathSeparator, removeUndefinedKeys } from "../../common/utils/utils.js";
8
8
import { DEFAULT_COMPOSE_GLOB, DEFAULT_ENV_GLOB, parseEnvConfig, selectComposeFiles, selectEnvFiles } from "./stackUtils.js";
9
9
10
10
export type BuildFileStackOptions = FilesOnServerConfig & { logger: Logger };
···
21
21
pollForUpdate,
22
22
server,
23
23
hostParentPath,
24
+
composeFiles = [],
25
+
projectName,
24
26
writeEnv = false,
25
27
} = options;
26
28
···
36
38
37
39
try {
38
40
stack = {
39
-
name: folderName,
41
+
name: projectName ?? folderName,
40
42
config: {
41
43
server,
42
44
run_directory: join(hostParentPath, folderName),
···
48
50
}
49
51
};
50
52
51
-
stack.config.file_paths = await selectComposeFiles(composeFileGlob, path, logger);
53
+
if(composeFiles.length > 0) {
54
+
stack.config.file_paths = composeFiles.map(x => removeRootPathSeparator(x.replace(path, '')));
55
+
} else {
56
+
stack.config.file_paths = await selectComposeFiles(composeFileGlob, path, logger);
57
+
}
58
+
52
59
53
60
stack.config = {
54
61
...stack.config,
···
68
75
throw new Error(`Error occurred while processing Stack for folder ${folderName}`, { cause: e });
69
76
} finally {
70
77
if (logJson) {
71
-
logger.debug(`Stack Config: ${JSON.stringify(stack)}}`);
78
+
logger.debug(`Stack Config: ${JSON.stringify(stack)}`);
72
79
}
73
80
}
74
81
}
+69
-54
src/builders/stack/gitStack.ts
+69
-54
src/builders/stack/gitStack.ts
···
1
1
import { Logger, childLogger } from "@foxxmd/logging";
2
-
import { FilesOnServerConfig, GitStackConfig, GitStackLinkedConfig, GitStackStandaloneConfig } from "../../common/infrastructure/config/stackConfig.js";
2
+
import { GitStackConfig } from "../../common/infrastructure/config/stackConfig.js";
3
3
import { _PartialStackConfig } from 'komodo_client/dist/types.js';
4
-
import { parse, ParsedPath, sep, join } from 'path';
4
+
import { parse, ParsedPath, sep, join, dirname, resolve } from 'path';
5
5
import { TomlStack } from "../../common/infrastructure/tomlObjects.js";
6
-
import { dirHasGitConfig, findFilesRecurive, readDirectories, readText, sortComposePaths } from "../../common/utils/io.js";
7
-
import { stripIndents } from "common-tags";
8
-
import { isDebugMode, removeUndefinedKeys } from "../../common/utils/utils.js";
6
+
import { findPathRecuriveParently } from "../../common/utils/io.js";
7
+
import { isDebugMode, removeRootPathSeparator, removeUndefinedKeys } from "../../common/utils/utils.js";
9
8
import { DEFAULT_COMPOSE_GLOB, DEFAULT_ENV_GLOB, parseEnvConfig, selectComposeFiles, selectEnvFiles } from "./stackUtils.js";
10
9
import { detectGitRepo, GitRepoData, komodoRepoFromRemote, matchGitDataWithKomodo, matchRemote, RemoteInfo } from "../../common/utils/git.js";
11
10
import { SimpleError } from "../../common/errors.js";
12
-
import { readFile } from "fs/promises";
13
11
14
12
export type BuildGitStackOptions = GitStackConfig & { logger: Logger };
15
13
···
24
22
autoUpdate,
25
23
pollForUpdate,
26
24
server,
27
-
inMonorepo = false,
28
-
hostParentPath,
25
+
projectName,
26
+
composeFiles = [],
29
27
writeEnv = false,
30
28
} = options
31
29
32
-
let gitStackConfig: _PartialStackConfig;
30
+
let gitStackConfig: _PartialStackConfig = {}
33
31
34
32
const pathInfo: ParsedPath = parse(path);
35
33
36
34
const folderName = `${pathInfo.name}${pathInfo.ext !== '' ? pathInfo.ext : ''}`;
37
35
38
-
const monoRepoPath = inMonorepo ? (hostParentPath !== undefined ? join(hostParentPath, folderName) : undefined) : undefined;
36
+
let repoRunDir: string;
39
37
40
38
const logger = childLogger(options.logger, [folderName, 'Git']);
41
39
42
-
if (inMonorepo === false) {
40
+
let nonGitReasons: string[] = [];
43
41
44
-
let gitData: GitRepoData;
45
-
try {
46
-
gitData = await detectGitRepo(path);
47
-
} catch (e) {
48
-
if (e instanceof SimpleError) {
49
-
throw new SimpleError(`Unable to parse path as git repo: ${e.message}`);
50
-
} else {
51
-
throw e;
42
+
let gitData: GitRepoData;
43
+
try {
44
+
gitData = await detectGitRepo(path);
45
+
logger.info(`Current directory is a Git repo: Branch ${gitData[0].branch} | Remote ${gitData[1].remote} | URL ${gitData[1].url}`);
46
+
} catch (e) {
47
+
if (e instanceof SimpleError) {
48
+
nonGitReasons.push(`Current dir is not a git repo => ${e.message}`);
49
+
} else {
50
+
throw e;
51
+
}
52
+
}
53
+
54
+
if(gitData === undefined) {
55
+
const parentGitPath = await findPathRecuriveParently(path, '.git');
56
+
if (parentGitPath !== undefined) {
57
+
const parentGitDir = dirname(parentGitPath)
58
+
try {
59
+
gitData = await detectGitRepo(parentGitDir);
60
+
logger.info(`Detected parent path ${parentGitDir} is a Git repo: Branch ${gitData[0].branch} | Remote ${gitData[1].remote} | URL ${gitData[1].url}`);
61
+
logger.info('Will treat current directory as the run directory for this repo');
62
+
repoRunDir = removeRootPathSeparator(path.replace(parentGitDir, ''));
63
+
gitStackConfig.run_directory = repoRunDir;
64
+
} catch (e) {
65
+
if (e instanceof SimpleError) {
66
+
// really shouldn't get here...
67
+
nonGitReasons.push(`Parent dir ${parentGitDir} is not a git repo => ${e.message}`);
68
+
} else {
69
+
throw e;
70
+
}
52
71
}
53
72
}
54
-
logger.info(`Folder has tracked branch '${gitData[0].branch}' with valid remote '${gitData[1].remote}'`);
73
+
}
74
+
75
+
if(gitData === undefined) {
76
+
throw new SimpleError(`Could not parse as a git repo: ${nonGitReasons.join(' | ')}`);
77
+
}
55
78
56
-
const [provider, repo, repoHint] = await matchGitDataWithKomodo(gitData);
79
+
const [provider, repo, repoHint] = await matchGitDataWithKomodo(gitData);
80
+
if (repo === undefined) {
81
+
logger.verbose(`No linked repo because ${repoHint}`);
82
+
const [domain, repo] = komodoRepoFromRemote(gitData[1].url);
57
83
if (repo === undefined) {
58
-
logger.verbose(`No linked repo because ${repoHint}`);
59
-
const [domain, repo] = komodoRepoFromRemote(gitData[1].url);
60
-
if (repo === undefined) {
61
-
throw new Error(`Could not parse repo from Remote URL ${gitData[1].url}`);
62
-
}
63
-
logger.debug(`Parsed Repo '${repo}' with ${provider?.domain !== undefined ? `provider` : 'URL'} domain '${provider?.domain ?? domain}' from Remote URL ${gitData[1].url}`);
64
-
if (provider?.username !== undefined) {
65
-
logger.debug(`Using provider username ${provider.username}`);
66
-
}
67
-
gitStackConfig = {
68
-
git_provider: provider?.domain ?? domain,
69
-
git_account: provider?.username,
70
-
repo
71
-
};
72
-
} else {
73
-
logger.verbose(`Using linked repo ${repo.name}}`);
74
-
gitStackConfig = {
75
-
linked_repo: repo.name,
76
-
}
84
+
throw new Error(`Could not parse repo from Remote URL ${gitData[1].url}`);
85
+
}
86
+
logger.debug(`Parsed Repo '${repo}' with ${provider?.domain !== undefined ? `provider` : 'URL'} domain '${provider?.domain ?? domain}' from Remote URL ${gitData[1].url}`);
87
+
if (provider?.username !== undefined) {
88
+
logger.debug(`Using provider username ${provider.username}`);
77
89
}
90
+
gitStackConfig = {
91
+
...gitStackConfig,
92
+
git_provider: provider?.domain ?? domain,
93
+
git_account: provider?.username,
94
+
repo
95
+
};
78
96
} else {
79
-
gitStackConfig = removeUndefinedKeys(
80
-
{
81
-
linked_repo: (options as GitStackLinkedConfig).linked_repo,
82
-
git_provider: (options as GitStackStandaloneConfig).git_provider,
83
-
git_account: (options as GitStackStandaloneConfig).git_account,
84
-
repo: (options as GitStackStandaloneConfig).repo,
85
-
run_directory: hostParentPath !== undefined ? join(hostParentPath, folderName) : undefined
86
-
}
87
-
)
97
+
logger.verbose(`Using linked repo ${repo.name}}`);
98
+
gitStackConfig. linked_repo = repo.name;
88
99
}
89
100
90
101
let stack: TomlStack;
···
92
103
93
104
try {
94
105
stack = {
95
-
name: folderName,
106
+
name: projectName ?? folderName,
96
107
config: {
97
108
server,
98
109
registry_account: imageRegistryAccount,
···
103
114
}
104
115
};
105
116
106
-
const composePaths = await selectComposeFiles(composeFileGlob, path, logger);
107
-
if(composePaths !== undefined) {
108
-
stack.config.file_paths = composePaths.map(x => inMonorepo ? join(monoRepoPath, x) : x);
117
+
if(composeFiles.length > 0) {
118
+
stack.config.file_paths = composeFiles.map(x => removeRootPathSeparator(x.replace(path, '')));
119
+
} else {
120
+
const composePaths = await selectComposeFiles(composeFileGlob, path, logger);
121
+
if(composePaths !== undefined) {
122
+
stack.config.file_paths = composePaths;
123
+
}
109
124
}
110
125
111
126
stack.config = {
···
113
128
...await(parseEnvConfig(path, {
114
129
envFileGlob,
115
130
writeEnv,
116
-
pathPrefix: monoRepoPath,
131
+
pathPrefix: repoRunDir,
117
132
komodoEnvName,
118
133
logger
119
134
}))
+72
-118
src/builders/stack/stackBuilder.ts
+72
-118
src/builders/stack/stackBuilder.ts
···
1
-
import { AnyStackConfig, FilesOnServerConfig, GitStackConfig } from "../../common/infrastructure/config/stackConfig.js";
1
+
import { AnyStackConfig } from "../../common/infrastructure/config/stackConfig.js";
2
2
import { TomlStack } from "../../common/infrastructure/tomlObjects.js";
3
-
import { promises } from 'fs';
4
-
import { findFolders, pathExistsAndIsReadable } from "../../common/utils/io.js";
3
+
import { findFolders } from "../../common/utils/io.js";
5
4
import { childLogger, Logger } from "@foxxmd/logging";
6
-
import { formatIntoColumns, isUndefinedOrEmptyString, parseBool } from "../../common/utils/utils.js";
7
-
import { detectGitRepo, GitRepoData, komodoRepoFromRemote, matchGitDataWithKomodo } from "../../common/utils/git.js";
5
+
import { formatIntoColumns, parseBool } from "../../common/utils/utils.js";
8
6
import { buildGitStack } from "./gitStack.js";
9
7
import { join as joinPath, parse, ParsedPath } from 'path';
10
-
import { DEFAULT_COMPOSE_GLOB, DEFAULT_ENV_GLOB } from "./stackUtils.js";
11
8
import { buildFileStack } from "./filesOnServer.js";
12
9
import { SimpleError } from "../../common/errors.js";
13
-
import { DEFAULT_GLOB_FOLDER } from "../../common/infrastructure/atomic.js";
10
+
import { DEFAULT_GLOB_FOLDER, DirectoryConfig, StackCandidateCompose, StackSourceOfTruth } from "../../common/infrastructure/atomic.js";
14
11
import { DockerApi } from "../../common/dockerApi.js";
12
+
import { consolidateComposeStacks } from "./composeUtils.js";
15
13
16
-
export const buildStacksFromPath = async (path: string, options: AnyStackConfig, parentLogger: Logger): Promise<TomlStack[]> => {
14
+
export class StackBuilder {
17
15
18
-
const {
19
-
composeFileGlob = DEFAULT_COMPOSE_GLOB,
20
-
envFileGlob = DEFAULT_ENV_GLOB,
21
-
folderGlob,
22
-
ignoreFolderGlob
23
-
} = options;
16
+
logger: Logger;
17
+
stackConfigOptions: AnyStackConfig;
24
18
25
-
let stackOptions = {
26
-
...options,
27
-
writeEnv: parseBool(process.env.WRITE_ENV, false)
28
-
};
19
+
dirData: DirectoryConfig;
29
20
30
-
const logger = childLogger(parentLogger, 'Stacks');
21
+
composeCandidateStacks: StackCandidateCompose[] = [];
31
22
32
-
const docker = new DockerApi();
33
-
const containers = await docker.getContainers({label: 'com.docker.compose.project'})
23
+
docker: DockerApi;
34
24
35
-
let topDir: string;
36
-
try {
37
-
topDir = await promises.realpath(path);
38
-
logger.info(`Top Dir: ${path} -> Resolved: ${topDir}`);
39
-
pathExistsAndIsReadable(topDir)
40
-
} catch (e) {
41
-
throw new Error(`Could not access ${path}.${parseBool(process.env.IS_DOCKER) ? ' This is the path *in container* that is read so make sure you have mounted it on the host!' : ''}`);
25
+
constructor(stacksConfig: AnyStackConfig, dirs: DirectoryConfig, logger: Logger) {
26
+
this.stackConfigOptions = stacksConfig;
27
+
this.dirData = dirs;
28
+
this.logger = childLogger(logger, 'Stacks');
29
+
this.docker = new DockerApi();
42
30
}
43
31
44
-
let gitData: GitRepoData;
45
-
try {
46
-
gitData = await detectGitRepo(path);
47
-
logger.verbose(`Detected top-level dir ${topDir} is a Git repo: Branch ${gitData[0].branch} | Remote ${gitData[1].remote} | URL ${gitData[1].url}`);
48
-
logger.verbose('Will treat this as a monorepo -- all subfolders will be built as Git-Repo Stacks using the same repo with Run Directory relative to repo root.');
49
-
} catch (e) {
50
-
if (e instanceof SimpleError) {
51
-
logger.verbose(`Top-level dir is not a git repo: ${e.message}`);
52
-
logger.verbose('Subfolders will be individually detected as Git repo or files-on-server Stacks')
53
-
} else {
54
-
throw e;
32
+
parseComposeProjects = async () => {
33
+
const containers = await this.docker.getContainers({ label: 'com.docker.compose.project' });
34
+
if (containers.length > 0) {
35
+
this.composeCandidateStacks = consolidateComposeStacks(containers, this.logger);
55
36
}
56
37
}
57
38
58
-
let stacksDir: string = topDir;
59
-
if (!isUndefinedOrEmptyString(process.env.GIT_STACKS_DIR) && gitData !== undefined) {
60
-
try {
61
-
stacksDir = await promises.realpath(joinPath(topDir, process.env.GIT_STACKS_DIR));
62
-
logger.info(`Git Stack Dir: ${process.env.GIT_STACKS_DIR} -> Resolved: ${stacksDir}`);
63
-
pathExistsAndIsReadable(stacksDir)
64
-
} catch (e) {
65
-
throw new Error(`Could not access ${stacksDir}.${parseBool(process.env.IS_DOCKER) ? ' This is the path *in container* that is read so make sure you have mounted it on the host!' : ''}`);
66
-
}
67
-
}
39
+
buildStacks = async (sourceOfTruth: StackSourceOfTruth): Promise<TomlStack[]> => {
68
40
69
-
let stacks: TomlStack[] = [];
41
+
this.logger.info(`Using ${sourceOfTruth === 'compose' ? ' parsed Compose projects' : 'child folders in SCAN_DIR'} to generate Stacks.`);
70
42
71
-
const dirs = await findFolders(stacksDir, folderGlob, ignoreFolderGlob)
72
-
const folderPaths = dirs.map(x => joinPath(stacksDir, x));
43
+
await this.parseComposeProjects();
73
44
74
-
logger.info(`Folder Glob: ${folderGlob ?? DEFAULT_GLOB_FOLDER}`);
75
-
logger.info(`Folder Ignore Glob${ignoreFolderGlob === undefined ? ' N/A ' : `: ${DEFAULT_GLOB_FOLDER}`}`);
76
-
logger.info(`Compose File Glob: ${composeFileGlob}`);
77
-
logger.info(`Env Glob: ${envFileGlob}`);
78
-
logger.info(`Processing Stacks for ${dirs.length} folders in ${stacksDir}:\n${formatIntoColumns(dirs, 3)}`);
45
+
const {
46
+
folderGlob,
47
+
ignoreFolderGlob
48
+
} = this.stackConfigOptions;
79
49
80
-
let hostParentPathVerified = false;
81
-
82
-
if (gitData !== undefined) {
50
+
let folderPaths: string[] = [];
83
51
84
-
let gitStackConfig: Partial<GitStackConfig> = {
85
-
inMonorepo: true
86
-
}
87
-
try {
88
-
const [provider, linkedRepo, repoHint] = await matchGitDataWithKomodo(gitData);
89
-
if (repoHint !== undefined) {
90
-
logger.warn(`All Stacks will be built without a linked repo: ${repoHint}`);
91
-
}
92
-
const [domain, repo] = komodoRepoFromRemote(gitData[1].url)
93
-
if (repo === undefined) {
94
-
gitStackConfig = {
95
-
...options,
96
-
git_provider: provider?.domain ?? domain,
97
-
git_account: provider?.username,
98
-
repo
99
-
};
100
-
} else {
101
-
gitStackConfig = {
102
-
...options,
103
-
linked_repo: linkedRepo.name,
52
+
if (sourceOfTruth === 'compose') {
53
+
for (const c of this.composeCandidateStacks) {
54
+
if (!c.workingDir.includes(this.dirData.host)) {
55
+
this.logger.warn(`Compose project ${c.projectName} working dir '${c.workingDir}' is not present in Host Dir, cannot use project`);
56
+
continue;
104
57
}
58
+
const convertedPath = c.workingDir.replace(this.dirData.host, this.dirData.mount);
59
+
this.logger.debug(`Compose project ${c.projectName} => Substituting in-environment path '${convertedPath}' for working dir '${c.workingDir}'`);
60
+
folderPaths.push(convertedPath);
105
61
}
106
-
} catch (e) {
107
-
throw new Error('Cannot use top-level git for Stacks', { cause: e });
108
-
}
62
+
this.logger.verbose(`Got ${folderPaths.length} valid compose project directories`);
63
+
} else {
64
+
this.logger.info(`Folder Glob: ${folderGlob ?? DEFAULT_GLOB_FOLDER}`);
65
+
this.logger.info(`Folder Ignore Glob${ignoreFolderGlob === undefined ? ' N/A ' : `: ${DEFAULT_GLOB_FOLDER}`}`);
109
66
110
-
stackOptions = {
111
-
...stackOptions,
112
-
...gitStackConfig,
113
-
inMonorepo: true
67
+
const dirs = await findFolders(this.dirData.scan, folderGlob, ignoreFolderGlob)
68
+
folderPaths = dirs.map(x => joinPath(this.dirData.scan, x));
69
+
this.logger.verbose(`Got ${folderPaths.length} folders in ${this.dirData.scan}:\n${formatIntoColumns(dirs, 3)}`);
114
70
}
115
-
}
116
71
117
-
for (const f of folderPaths) {
72
+
let stacks: TomlStack[] = [];
73
+
74
+
let stackOptions = {
75
+
...this.stackConfigOptions,
76
+
writeEnv: parseBool(process.env.WRITE_ENV, false)
77
+
};
78
+
79
+
for (const f of folderPaths) {
80
+
81
+
let composeFiles: string[] = [];
82
+
let projectName: string;
118
83
119
-
const pathInfo: ParsedPath = parse(f);
120
-
const folderLogger = childLogger(logger, `${pathInfo.name}${pathInfo.ext !== '' ? pathInfo.ext : ''}`);
84
+
const folderStackOptions = {...stackOptions};
121
85
122
-
if (gitData !== undefined) {
123
-
try {
124
-
stacks.push(await buildGitStack(f,
125
-
{
126
-
...stackOptions,
127
-
logger,
128
-
hostParentPath: stacksDir === topDir ? undefined : stacksDir.replace(topDir, '').replace(/^\//, '')
129
-
}));
130
-
} catch (e) {
131
-
folderLogger.error(new Error(`Unable to build Git Stack for folder ${f}`, { cause: e }));
86
+
const matchedProject = this.composeCandidateStacks.find(x => x.workingDir.replace(this.dirData.host, this.dirData.mount) === f);
87
+
if(matchedProject !== undefined) {
88
+
projectName = matchedProject.projectName;
89
+
composeFiles = matchedProject.composeFilePaths.map(x => x.replace(matchedProject.workingDir, ''));
90
+
folderStackOptions.projectName = projectName;
91
+
folderStackOptions.composeFiles = composeFiles;
132
92
}
133
-
} else {
93
+
94
+
95
+
const pathInfo: ParsedPath = parse(f);
96
+
const folderLogger = childLogger(this.logger, `${pathInfo.name}${pathInfo.ext !== '' ? pathInfo.ext : ''}`);
134
97
135
98
try {
136
99
stacks.push(await buildGitStack(f, {
137
-
inMonorepo: false,
138
-
...stackOptions,
139
-
logger
100
+
...folderStackOptions,
101
+
logger: this.logger,
140
102
}));
141
103
continue;
142
104
} catch (e) {
143
105
if (e instanceof SimpleError) {
144
-
if (e.message.includes('does not have a .git folder')) {
145
-
folderLogger.debug('Falling back to Files-On-Server, not a git repo');
146
-
} else {
147
-
folderLogger.verbose(`Falling back to Files-On-Server => ${e.message}`);
148
-
}
106
+
folderLogger.verbose(`Falling back to Files-On-Server => ${e.message}`);
149
107
} else {
150
108
folderLogger.error(new Error(`Unable to build Git Stack for folder ${f}`, { cause: e }));
151
109
continue;
152
110
}
153
111
}
154
112
155
-
if (!hostParentPathVerified) {
156
-
if (isUndefinedOrEmptyString(stackOptions.hostParentPath)) {
157
-
throw new Error('env HOST_PARENT_PATH is not set');
158
-
}
159
-
hostParentPathVerified = true;
160
-
}
161
113
try {
162
-
stacks.push(await buildFileStack(f, { ...stackOptions, logger }));
114
+
stacks.push(await buildFileStack(f, { ...folderStackOptions, hostParentPath: this.dirData.host, logger: this.logger }));
163
115
} catch (e) {
164
116
folderLogger.error(new Error(`Unable to build Files-On-Server Stack for folder ${f}`, { cause: e }));
165
117
}
118
+
166
119
}
120
+
121
+
return stacks;
167
122
}
168
-
return stacks;
169
123
}
+14
src/common/infrastructure/atomic.ts
+14
src/common/infrastructure/atomic.ts
···
15
15
16
16
export type StackDiscoveryMethod = 'compose' | 'monorepo' | 'folder';
17
17
18
+
export type StackSourceOfTruth = 'compose' | 'dir';
19
+
18
20
export interface StackCandidate {
19
21
path: string
20
22
discovered: StackDiscoveryMethod
···
26
28
workingDir: string
27
29
composeFilePaths: string[]
28
30
state: 'running' | string
31
+
}
32
+
33
+
export interface DirectoryConfigValues {
34
+
mountVal?: string
35
+
hostVal?: string
36
+
scanVal?: string
37
+
}
38
+
39
+
export interface DirectoryConfig {
40
+
mount: string
41
+
host: string
42
+
scan: string
29
43
}
+2
src/common/infrastructure/config/stackConfig.ts
+2
src/common/infrastructure/config/stackConfig.ts
+75
-2
src/common/utils/io.ts
+75
-2
src/common/utils/io.ts
···
1
1
import { accessSync, constants, promises } from "fs";
2
-
import pathUtil from "path";
2
+
import pathUtil, { join, dirname } from "path";
3
3
import { glob, GlobOptionsWithFileTypesUnset } from 'glob';
4
4
import clone from 'clone';
5
-
import { DEFAULT_GLOB_FOLDER } from "../infrastructure/atomic.js";
5
+
import { DEFAULT_GLOB_FOLDER, DirectoryConfig, DirectoryConfigValues } from "../infrastructure/atomic.js";
6
+
import { parseBool } from "./utils.js";
6
7
7
8
export async function writeFile(path: any, text: any) {
8
9
try {
···
79
80
}
80
81
}
81
82
83
+
export const findPathRecuriveParently = async (fromDir: string, path: string) => {
84
+
85
+
let currDirectory = fromDir;
86
+
let parentDirectory = fromDir;
87
+
let firstRun = true;
88
+
89
+
// https://hals.app/blog/recursively-read-parent-folder-nodejs/
90
+
while (firstRun || currDirectory !== parentDirectory) {
91
+
92
+
firstRun = false;
93
+
94
+
const files = await glob(path, {
95
+
cwd: parentDirectory,
96
+
});
97
+
98
+
if(files.length > 0) {
99
+
return join(parentDirectory, path);
100
+
}
101
+
102
+
// The trick is here:
103
+
// Using path.dirname() of a directory returns the parent directory!
104
+
currDirectory = parentDirectory
105
+
parentDirectory = dirname(parentDirectory);
106
+
}
107
+
108
+
return undefined;
109
+
}
110
+
82
111
export const sortComposePaths = (p: string[]): string[] => {
83
112
const paths = clone(p);
84
113
paths.sort((a, b) => {
···
144
173
145
174
export const dirHasGitConfig = (paths: string[]): boolean => {
146
175
return paths.some(x => x === '.git');
176
+
}
177
+
178
+
export const parseDirectoryConfig = async (dirConfig: DirectoryConfigValues = {}): Promise<[DirectoryConfigValues, DirectoryConfig]> => {
179
+
const {
180
+
mountVal = process.env.MOUNT_DIR,
181
+
hostVal = process.env.HOST_DIR ?? mountVal,
182
+
scanVal = process.env.SCAN_DIR ?? mountVal
183
+
} = dirConfig;
184
+
185
+
const configVal = {
186
+
mountVal,
187
+
hostVal,
188
+
scanVal
189
+
};
190
+
191
+
let mountDir: string,
192
+
hostDir: string,
193
+
scanDir: string;
194
+
195
+
try {
196
+
mountDir = await promises.realpath(mountVal);
197
+
pathExistsAndIsReadable(mountDir)
198
+
} catch (e) {
199
+
throw new Error(`Could not access directory for MOUNT_DIR ${mountVal}.${parseBool(process.env.IS_DOCKER) ? ' This is the path *in container* that is read so make sure you have mounted it on the host!' : ''}`);
200
+
}
201
+
202
+
if (hostVal === mountVal) {
203
+
hostDir = mountDir;
204
+
} else {
205
+
hostDir = hostVal;
206
+
}
207
+
208
+
if (scanVal === mountVal) {
209
+
scanDir = mountDir;
210
+
} else {
211
+
try {
212
+
scanDir = await promises.realpath(join(mountDir, scanVal));
213
+
pathExistsAndIsReadable(scanDir)
214
+
} catch (e) {
215
+
throw new Error(`Could not access directory for SCAN_DIR '${scanVal}' -- this should be the *relative* path from HOST_DIR that we look for stack-folders in.`);
216
+
}
217
+
}
218
+
219
+
return [configVal, { host: hostDir, scan: scanDir, mount: mountDir }]
147
220
}
+9
-1
src/common/utils/utils.ts
+9
-1
src/common/utils/utils.ts
···
1
1
import { searchAndReplace } from "@foxxmd/regex-buddy-core";
2
2
import { stripIndents } from "common-tags";
3
+
import path, {sep} from 'path';
3
4
4
5
export const isDebugMode = (): boolean => process.env.DEBUG_MODE === 'true';
5
6
···
26
27
if(Array.isArray(obj[key])) {
27
28
newObj[key] = obj[key];
28
29
} else if (obj[key] === Object(obj[key])) {
29
-
newObj[key] = removeUndefinedKeys(obj[key]);
30
+
newObj[key] = obj[key];
30
31
} else if (obj[key] !== undefined) {
31
32
newObj[key] = obj[key];
32
33
}
···
91
92
});
92
93
93
94
return transformed;
95
+
}
96
+
97
+
export const removeRootPathSeparator = (pathStr: string): string => {
98
+
if(pathStr.at(0) === sep) {
99
+
return pathStr.substring(1);
100
+
}
101
+
return pathStr;
94
102
}
+20
-4
src/index.ts
+20
-4
src/index.ts
···
16
16
import { exportToFile } from './exporters/exportToFile.js';
17
17
import { exportToSync } from './exporters/exportToApiSync.js';
18
18
import { getDefaultKomodoApi } from './common/utils/komodo.js';
19
-
import { buildStacksFromPath } from './builders/stack/stackBuilder.js';
19
+
import { StackBuilder } from './builders/stack/stackBuilder.js';
20
+
import { parseDirectoryConfig } from './common/utils/io.js';
21
+
import { DirectoryConfig, DirectoryConfigValues } from './common/infrastructure/atomic.js';
20
22
21
23
dayjs.extend(utc)
22
24
dayjs.extend(timezone);
···
34
36
initLogger.error(appError);
35
37
}
36
38
});
37
-
38
-
const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
39
39
40
40
try {
41
41
···
56
56
57
57
getDefaultKomodoApi(logger);
58
58
59
+
let dirData: [DirectoryConfigValues, DirectoryConfig];
60
+
try {
61
+
dirData = await parseDirectoryConfig();
62
+
logger.info(`Mount Dir : ${dirData[0].mountVal} -> Resolved: ${dirData[1].mount}`);
63
+
logger.info(`Host Dir : ${dirData[0].hostVal} -> Resolved: ${dirData[1].host}`);
64
+
logger.info(`Scan Dir : ${dirData[0].scanVal} -> Resolved: ${dirData[1].scan}`);
65
+
} catch (e) {
66
+
throw new Error('Could not parse required directories', {cause: e});
67
+
}
68
+
59
69
const importOptions: CommonImportOptions = {
60
70
server: process.env.SERVER_NAME,
61
71
imageRegistryProvider: process.env.IMAGE_REGISTRY_PROVIDER,
···
79
89
hostParentPath: process.env.HOST_PARENT_PATH
80
90
};
81
91
92
+
const stackBuilder = new StackBuilder(filesOnServerConfig, dirData[1], logger);
93
+
82
94
let stacks: TomlStack[] = [];
83
-
stacks = await buildStacksFromPath(process.env.FILES_ON_SERVER_DIR, filesOnServerConfig, logger);
95
+
let stackModeVal = (process.env.STACKS_FROM ?? 'dir').toLocaleLowerCase();
96
+
if(stackModeVal !== 'compose' && stackModeVal !== 'dir') {
97
+
throw new Error(`STACKS_FROM must be either 'compose' or 'dir'`);
98
+
}
99
+
stacks = await stackBuilder.buildStacks(stackModeVal);//await buildStacksFromPath(process.env.FILES_ON_SERVER_DIR, filesOnServerConfig, logger);
84
100
85
101
if (stacks.length === 0) {
86
102
logger.info('No Stacks found! Nothing to do.');
+20
-3
tests/filesOnServer.test.ts
+20
-3
tests/filesOnServer.test.ts
···
1
1
import { describe, it } from 'mocha';
2
2
import chai, { expect } from 'chai';
3
3
import withLocalTmpDir from 'with-local-tmp-dir';
4
-
import { mkdir, writeFile, chmod, constants } from 'node:fs/promises';
4
+
import { mkdir, writeFile } from 'node:fs/promises';
5
5
import path from "path";
6
6
import { loggerTest } from "@foxxmd/logging";
7
7
import { buildFileStack, BuildFileStackOptions } from '../src/builders/stack/filesOnServer.js';
8
-
import { FilesOnServerConfig } from '../src/common/infrastructure/config/stackConfig.js';
9
-
import { stripIndents } from 'common-tags';
10
8
11
9
const DEFAULT_FOS_PATH = '/my/cool'
12
10
const DEFAULT_SERVER = 'test-server';
···
115
113
116
114
expect(data.config.file_paths).to.not.be.undefined;
117
115
expect(data.config.file_paths.length).eq(2);
116
+
}, { unsafeCleanup: true });
117
+
});
118
+
119
+
it(`includes all compose files when provided as arg`, async function () {
120
+
await withLocalTmpDir(async () => {
121
+
122
+
const dir = path.join(process.cwd(), 'test_1');
123
+
await mkdir(path.join(dir, 'nested'), {recursive: true});
124
+
await writeFile(path.join(dir, 'nested', 'docker-compose.yaml'), '');
125
+
126
+
const data = await buildFileStack(dir, {
127
+
...defaultFOSConfig,
128
+
composeFileGlob: '**/{compose,docker-compose}*.yaml',
129
+
composeFiles: [path.join(dir, 'nested', 'docker-compose.yaml')]
130
+
});
131
+
132
+
expect(data.config.file_paths).to.not.be.undefined;
133
+
expect(data.config.file_paths.length).eq(1);
134
+
expect(data.config.file_paths[0]).eq('nested/docker-compose.yaml')
118
135
}, { unsafeCleanup: true });
119
136
});
120
137
+147
-27
tests/git.test.ts
+147
-27
tests/git.test.ts
···
11
11
import git from 'isomorphic-git';
12
12
import fs from 'fs';
13
13
import { execa, Options } from 'execa';
14
+
import { faker } from '@faker-js/faker';
15
+
import { readText } from '../src/common/utils/io.js';
14
16
15
17
chai.use(chaiAsPromised);
16
18
···
215
217
});
216
218
});
217
219
218
-
// describe('#GitStack', function () {
220
+
describe('#GitStack', function () {
219
221
220
-
// it(`creates a valid stack`, async function () {
221
-
// await withLocalTmpDir(async () => {
222
-
// const dir = path.join(process.cwd(), 'test_1');
223
-
// await mkdir(dir);
224
-
// const localDir = path.join(dir, 'local');
225
-
// //await mkdir(localDir);
226
-
// const remoteDir = path.join(dir, 'remote');
227
-
// await mkdir(remoteDir);
228
-
// await git.init({ fs, dir: remoteDir });
229
-
// await git.commit({
230
-
// fs,
231
-
// dir: remoteDir,
232
-
// author: {
233
-
// name: 'Mr. Test',
234
-
// email: 'mrtest@example.com',
235
-
// },
236
-
// message: 'Added the a.txt file'
237
-
// });
238
-
// // cannot yet clone local repo with isomorphic git
239
-
// // https://github.com/isomorphic-git/isomorphic-git/issues/1263
240
-
// await execa({ cwd: dir })`git clone remote local`
241
-
// await expect(buildGitStack(localDir, defaultGitStandaloneConfig)).to.eventually.be.fulfilled;
242
-
// }, { unsafeCleanup: true });
243
-
// });
222
+
it(`creates a valid stack from top-dir git`, async function () {
223
+
await withLocalTmpDir(async () => {
224
+
const dir = path.join(process.cwd(), 'test_1');
225
+
await mkdir(dir);
226
+
await git.init({ fs, dir });
227
+
await git.commit({
228
+
fs,
229
+
dir,
230
+
author: {
231
+
name: 'Mr. Test',
232
+
email: 'mrtest@example.com',
233
+
},
234
+
message: 'Added the a.txt file'
235
+
});
236
+
const rc = await readText(path.join(dir, '.git', 'config'));
237
+
const [fc, fakeData] = fakeGitConfig();
238
+
await writeFile(path.join(dir, '.git', 'config'), `${rc}\n${fc}`);
244
239
245
-
// });
246
-
});
240
+
const stack = await buildGitStack(dir, defaultGitStandaloneConfig);
241
+
expect(stack.config.repo).eq(`${fakeData.user}/${fakeData.repo}`);
242
+
243
+
}, { unsafeCleanup: true });
244
+
});
245
+
246
+
it(`uses relative path for compose files`, async function () {
247
+
await withLocalTmpDir(async () => {
248
+
const dir = path.join(process.cwd(), 'test_1');
249
+
await mkdir(dir);
250
+
await mkdir(path.join(dir, 'nested'))
251
+
await writeFile(path.join(dir, 'nested', 'compose.yml'), '');
252
+
await git.init({ fs, dir });
253
+
await git.commit({
254
+
fs,
255
+
dir,
256
+
author: {
257
+
name: 'Mr. Test',
258
+
email: 'mrtest@example.com',
259
+
},
260
+
message: 'Added the a.txt file'
261
+
});
262
+
263
+
const rc = await readText(path.join(dir, '.git', 'config'));
264
+
const [fc, fakeData] = fakeGitConfig();
265
+
await writeFile(path.join(dir, '.git', 'config'), `${rc}\n${fc}`);
266
+
267
+
const stack = await buildGitStack(dir, defaultGitStandaloneConfig);
268
+
expect(stack.config.file_paths).to.not.be.undefined;
269
+
expect(stack.config.file_paths[0]).eq('nested/compose.yml');
270
+
271
+
}, { unsafeCleanup: true });
272
+
});
273
+
274
+
it(`sets relative path from parent when git is above dir`, async function () {
275
+
await withLocalTmpDir(async () => {
276
+
const dir = path.join(process.cwd(), 'test_1');
277
+
await mkdir(dir);
278
+
await mkdir(path.join(dir, 'nested', 'docker'), {recursive: true})
279
+
await writeFile(path.join(dir, 'nested', 'docker', 'compose.yml'), '');
280
+
await git.init({ fs, dir });
281
+
await git.commit({
282
+
fs,
283
+
dir,
284
+
author: {
285
+
name: 'Mr. Test',
286
+
email: 'mrtest@example.com',
287
+
},
288
+
message: 'Added the a.txt file'
289
+
});
290
+
291
+
const rc = await readText(path.join(dir, '.git', 'config'));
292
+
const [fc, fakeData] = fakeGitConfig();
293
+
await writeFile(path.join(dir, '.git', 'config'), `${rc}\n${fc}`);
294
+
295
+
const stack = await buildGitStack(path.join(dir, 'nested'), defaultGitStandaloneConfig);
296
+
expect(stack.config.run_directory).to.not.be.undefined;
297
+
expect(stack.config.run_directory).eq('nested');
298
+
expect(stack.config.file_paths).to.not.be.undefined;
299
+
expect(stack.config.file_paths[0]).eq('docker/compose.yml');
300
+
301
+
}, { unsafeCleanup: true });
302
+
});
303
+
304
+
it(`determines relative path for compose files when argument provided`, async function () {
305
+
await withLocalTmpDir(async () => {
306
+
const dir = path.join(process.cwd(), 'test_1');
307
+
await mkdir(dir);
308
+
await mkdir(path.join(dir, 'nested', 'docker'), {recursive: true})
309
+
await writeFile(path.join(dir, 'nested', 'docker', 'compose.yml'), '');
310
+
await git.init({ fs, dir });
311
+
await git.commit({
312
+
fs,
313
+
dir,
314
+
author: {
315
+
name: 'Mr. Test',
316
+
email: 'mrtest@example.com',
317
+
},
318
+
message: 'Added the a.txt file'
319
+
});
320
+
321
+
const rc = await readText(path.join(dir, '.git', 'config'));
322
+
const [fc, fakeData] = fakeGitConfig();
323
+
await writeFile(path.join(dir, '.git', 'config'), `${rc}\n${fc}`);
324
+
325
+
const stack = await buildGitStack(path.join(dir, 'nested'), {
326
+
...defaultGitStandaloneConfig,
327
+
composeFiles: [path.join(dir, 'nested', 'docker', 'compose.yml')]
328
+
});
329
+
expect(stack.config.run_directory).to.not.be.undefined;
330
+
expect(stack.config.run_directory).eq('nested');
331
+
expect(stack.config.file_paths).to.not.be.undefined;
332
+
expect(stack.config.file_paths[0]).eq('docker/compose.yml');
333
+
334
+
}, { unsafeCleanup: true });
335
+
});
336
+
337
+
});
338
+
});
339
+
340
+
interface FakeGitConfig {
341
+
user?: string
342
+
repo?: string
343
+
domain?: string
344
+
}
345
+
const fakeGitConfig = (options: FakeGitConfig = {}): [string, Required<FakeGitConfig>] => {
346
+
347
+
const {
348
+
domain = 'github.com',
349
+
user = faker.internet.userName(),
350
+
repo = `${faker.animal.type()}-${faker.word.adjective()}`
351
+
} = options;
352
+
353
+
const used: Required<FakeGitConfig> = {
354
+
domain,
355
+
user,
356
+
repo
357
+
};
358
+
359
+
const configContents = `[remote "origin"]
360
+
url = https://${domain}/${user}/${repo}
361
+
fetch = +refs/heads/*:refs/remotes/origin/*
362
+
[branch "master"]
363
+
remote = origin
364
+
merge = refs/heads/master`;
365
+
return [configContents, used]
366
+
}