onedev/k8s-helper

Refactor to pull out some common functionality
Robin Shen committed 3 years ago

5be88909

1 parent dc267a47
Release

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

Total 5 files Show one by one

■ ■ ■ ■ ■ ■

src/main/java/io/onedev/k8shelper/Init.java

		skipped 4 lines
5	5
6	6		import com.google.common.base.Preconditions;
7	7
	8	+	import io.onedev.commons.utils.TaskLogger;
	9	+
8	10		public class Init {
9	11
10	12		private static final Logger logger = LoggerFactory.getLogger(Init.class);
		skipped 11 lines
22	24		logger.info(KubernetesHelper.LOG_END_MESSAGE);
23	25		System.exit(0);
24	26		} catch (Exception e) {
25		-	logger.error(KubernetesHelper.wrapWithAnsiError("Error executing init logic"), e);
	27	+	logger.error(TaskLogger.wrapWithAnsiError("Error executing init logic"), e);
26	28		logger.info(KubernetesHelper.LOG_END_MESSAGE);
27	29		System.exit(1);
28	30		}
		skipped 4 lines

■ ■ ■ ■ ■ ■

src/main/java/io/onedev/k8shelper/JobData.java

1	+	package io.onedev.k8shelper;
2	+
3	+	import java.io.Serializable;
4	+	import java.util.List;
5	+
6	+	public class JobData implements Serializable {
7	+
8	+	private static final long serialVersionUID = 1L;
9	+
10	+	private final String commitHash;
11	+
12	+	private final List<Action> actions;
13	+
14	+	public JobData(String commitHash, List<Action> actions) {
15	+	this.commitHash = commitHash;
16	+	this.actions = actions;
17	+	}
18	+
19	+	public String getCommitHash() {
20	+	return commitHash;
21	+	}
22	+
23	+	public List<Action> getActions() {
24	+	return actions;
25	+	}
26	+
27	+	}
28	+

■ ■ ■ ■ ■ ■

src/main/java/io/onedev/k8shelper/KubernetesHelper.java

		skipped 49 lines
50	50		import io.onedev.commons.utils.ExplicitException;
51	51		import io.onedev.commons.utils.FileUtils;
52	52		import io.onedev.commons.utils.PathUtils;
	53	+	import io.onedev.commons.utils.TaskLogger;
53	54		import io.onedev.commons.utils.command.Commandline;
54	55		import io.onedev.commons.utils.command.LineConsumer;
55	56
		skipped 26 lines
82	83		return new File("/onedev-build");
83	84		}
84	85
85		-	private static File getJobContextFile() {
86		-	return new File(getBuildHome(), "job-context");
	86	+	private static File getJobDataFile() {
	87	+	return new File(getBuildHome(), "job-data");
87	88		}
88	89
89	90		private static File getTrustCertsHome() {
		skipped 188 lines
278	279		@Override
279	280		public void consume(String line) {
280	281		if (!line.contains("Warning: use -cacerts option to access cacerts keystore"))
281		-	logger.error(wrapWithAnsiError(line));
	282	+	logger.error(TaskLogger.wrapWithAnsiError(line));
282	283		}
283	284
284	285		}).checkReturnCode();
		skipped 18 lines
303	304
304	305		@Override
305	306		public void consume(String line) {
306		-	logger.error(wrapWithAnsiError(line));
	307	+	logger.error(TaskLogger.wrapWithAnsiError(line));
307	308		}
308	309
309	310		};
		skipped 28 lines
338	339		git.execute(infoLogger, errorLogger).checkReturnCode();
339	340		}
340	341
341		-	@SuppressWarnings("unchecked")
342	342		public static void init(String serverUrl, String jobToken, boolean test) {
343	343		installJVMCert();
344	344
		skipped 28 lines
373	373		Lists.newArrayList("echo hello from container"));
374	374		}
375	375		} else {
376		-	WebTarget target = client.target(serverUrl).path("api/k8s/job-context");
	376	+	WebTarget target = client.target(serverUrl).path("api/k8s/job-data");
377	377		Invocation.Builder builder = target.request();
378	378		builder.header(HttpHeaders.AUTHORIZATION, BEARER + " " + jobToken);
379	379
380		-	logger.info("Retrieving job context from {}...", serverUrl);
	380	+	logger.info("Retrieving job data from {}...", serverUrl);
381	381
382		-	Map<String, Object> jobContext;
383		-	byte[] jobContextBytes;
	382	+	JobData jobData;
	383	+	byte[] jobDataBytes;
384	384		try (Response response = builder.post(
385	385		Entity.entity(getWorkspace().getAbsolutePath(), MediaType.APPLICATION_OCTET_STREAM))) {
386	386		checkStatus(response);
387		-	jobContextBytes = response.readEntity(byte[].class);
	387	+	jobDataBytes = response.readEntity(byte[].class);
388	388		}
389	389
390		-	FileUtils.writeByteArrayToFile(getJobContextFile(), jobContextBytes);
391		-	jobContext = SerializationUtils.deserialize(jobContextBytes);
	390	+	FileUtils.writeByteArrayToFile(getJobDataFile(), jobDataBytes);
	391	+	jobData = SerializationUtils.deserialize(jobDataBytes);
392	392
393	393		File workspace = getWorkspace();
394	394		File workspaceCache = null;
		skipped 36 lines
431	431
432	432		logger.info("Generating command scripts...");
433	433
434		-	List<Action> actions = (List<Action>) jobContext.get("actions");
435		-	CompositeExecutable entryExecutable = new CompositeExecutable(actions);
	434	+	CompositeExecutable entryExecutable = new CompositeExecutable(jobData.getActions());
436	435		entryExecutable.traverse(new LeafVisitor<Void>() {
437	436
438	437		@Override
		skipped 3 lines
442	441		List<String> setupCommands = new ArrayList<>();
443	442		if (isWindows()) {
444	443		setupCommands.add("@echo off");
445		-	setupCommands.add("xcopy /Y /S /K /Q /H /R C:\\Users\\%USERNAME%\\onedev\\* C:\\Users\\%USERNAME% > nul");
	444	+	setupCommands.add("xcopy /Y /S /K /Q /H /R C:\\Users\\%USERNAME%\\auth-info\\* C:\\Users\\%USERNAME% > nul");
446	445		} else {
447		-	setupCommands.add("cp -r -f -p /root/onedev/. /root");
	446	+	setupCommands.add("cp -r -f -p /root/auth-info/. /root");
448	447		}
449	448
450	449		for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {
		skipped 88 lines
539	538		.collect(Collectors.toList());
540	539		}
541	540
542		-	private static void checkStatus(Response response) {
	541	+	public static void checkStatus(Response response) {
543	542		int status = response.getStatus();
544	543		if (status != 200 && status != 204) {
545	544		String errorMessage = response.readEntity(String.class);
		skipped 7 lines
553	552		}
554	553		}
555	554
556		-	public static void cloneRepository(File workspace, String cloneUrl, String commitHash,
557		-	int cloneDepth, Commandline git, LineConsumer infoLogger, LineConsumer errorLogger) {
	555	+	public static void initRepositoryIfNecessary(Commandline git, LineConsumer infoLogger,
	556	+	LineConsumer errorLogger) {
558	557		git.clearArgs();
559		-	if (!new File(workspace, ".git").exists()) {
	558	+	if (!new File(git.workingDir(), ".git").exists()) {
560	559		git.addArgs("init", ".");
561	560		git.execute(new LineConsumer() {
562	561
		skipped 5 lines
568	567
569	568		}, errorLogger).checkReturnCode();
570	569		}
571		-
572		-	git.clearArgs();
573		-	git.addArgs("fetch", cloneUrl, "--force", "--quiet");
574		-	if (cloneDepth != 0)
575		-	git.addArgs("--depth=" + cloneDepth);
576		-	git.addArgs(commitHash);
577		-	git.execute(infoLogger, errorLogger).checkReturnCode();
	570	+	}
578	571
	572	+	public static void checkoutRepository(Commandline git, String commitHash,
	573	+	LineConsumer infoLogger, LineConsumer errorLogger) {
579	574		git.clearArgs();
580	575		git.addArgs("checkout", "--quiet", commitHash);
581	576		git.execute(infoLogger, errorLogger).checkReturnCode();
	577	+	}
582	578
583		-	if (new File(workspace, ".gitmodules").exists()) {
	579	+	public static void deinitSubmodulesIfNecessary(Commandline git,
	580	+	LineConsumer infoLogger, LineConsumer errorLogger) {
	581	+	if (new File(git.workingDir(), ".gitmodules").exists()) {
584	582		// deinit submodules in case submodule url is changed
585	583		git.clearArgs();
586	584		git.addArgs("submodule", "deinit", "--all", "--force", "--quiet");
		skipped 11 lines
598	596		}
599	597		}
600	598
601		-	private static Map<String, Object> readJobContext() {
602		-	byte[] jobContextBytes;
	599	+	public static void updateSubmodulesIfNecessary(Commandline git, int cloneDepth,
	600	+	LineConsumer infoLogger, LineConsumer errorLogger) {
	601	+	if (new File(git.workingDir(), ".gitmodules").exists()) {
	602	+	logger.info("Retrieving submodules...");
	603	+
	604	+	git.clearArgs();
	605	+	git.addArgs("submodule", "update", "--init", "--recursive", "--force", "--quiet");
	606	+	if (cloneDepth != 0)
	607	+	git.addArgs("--depth=" + cloneDepth);
	608	+	git.execute(infoLogger, new LineConsumer() {
	609	+
	610	+	@Override
	611	+	public void consume(String line) {
	612	+	if (line.contains("Submodule") && line.contains("registered for path")
	613	+	\|\| line.startsWith("From ") \|\| line.startsWith(" * branch")
	614	+	\|\| line.startsWith(" +") && line.contains("->")) {
	615	+	infoLogger.consume(line);
	616	+	} else {
	617	+	errorLogger.consume(line);
	618	+	}
	619	+	}
	620	+
	621	+	}).checkReturnCode();
	622	+	}
	623	+	}
	624	+
	625	+	public static void cloneRepository(Commandline git, String cloneUrl, String commitHash,
	626	+	int cloneDepth, LineConsumer infoLogger, LineConsumer errorLogger) {
	627	+	initRepositoryIfNecessary(git, infoLogger, errorLogger);
	628	+
	629	+	git.clearArgs();
	630	+	git.addArgs("fetch", cloneUrl, "--force", "--quiet");
	631	+	if (cloneDepth != 0)
	632	+	git.addArgs("--depth=" + cloneDepth);
	633	+	git.addArgs(commitHash);
	634	+	git.execute(infoLogger, errorLogger).checkReturnCode();
	635	+
	636	+	checkoutRepository(git, commitHash, infoLogger, errorLogger);
	637	+	deinitSubmodulesIfNecessary(git, infoLogger, errorLogger);
	638	+	}
	639	+
	640	+	private static JobData readJobData() {
	641	+	byte[] jobDataBytes;
603	642		try {
604		-	jobContextBytes = FileUtils.readFileToByteArray(getJobContextFile());
	643	+	jobDataBytes = FileUtils.readFileToByteArray(getJobDataFile());
605	644		} catch (IOException e) {
606	645		throw new RuntimeException(e);
607	646		}
608		-	return SerializationUtils.deserialize(jobContextBytes);
	647	+	return SerializationUtils.deserialize(jobDataBytes);
609	648		}
610	649
611		-	@SuppressWarnings("unchecked")
612	650		public static void sidecar(String serverUrl, String jobToken, boolean test) {
613	651		LeafHandler commandHandler = new LeafHandler() {
614	652
		skipped 65 lines
680	718		"this does not matter", Lists.newArrayList("this does not matter"), false);
681	719		executable.execute(commandHandler, Lists.newArrayList(1));
682	720		} else {
683		-	Map<String, Object> jobContext = readJobContext();
	721	+	JobData jobData = readJobData();
684	722
685		-	List<Action> actions = (List<Action>) jobContext.get("actions");
	723	+	List<Action> actions = jobData.getActions();
686	724
687	725		new CompositeExecutable(actions).execute(commandHandler, new ArrayList<>());
688	726
		skipped 66 lines
755	793
756	794		public static void checkoutCode(String serverUrl, String jobToken, String positionStr,
757	795		int cloneDepth, CloneInfo cloneInfo) throws IOException {
758		-	Map<String, Object> jobContext = readJobContext();
759		-	String commitHash = (String) jobContext.get("commitHash");
	796	+	JobData jobData = readJobData();
760	797
761	798		logger.info("Checking out code from {}...", cloneInfo.getCloneUrl());
762	799
		skipped 11 lines
774	811
775	812		cloneInfo.writeAuthData(userHome, git, infoLogger, errorLogger);
776	813
777		-	File mountedUserHome = new File(userHome, "onedev");
778		-	Commandline gitOfMountedUserHome = new Commandline("git");
779		-	gitOfMountedUserHome.environments().put("HOME", mountedUserHome.getAbsolutePath());
780		-
781		-	// Populate auth data here in case user want to do additional pull/push in other step container
782		-	cloneInfo.writeAuthData(mountedUserHome, gitOfMountedUserHome, infoLogger, errorLogger);
	814	+	// Also populate auth info into authInfoHome which will be shared
	815	+	// with other containers. The setup script of other contains will
	816	+	// move all auth data from authInfoHome into the user home so that
	817	+	// git pull/push can be done without asking for credentials
	818	+	File authInfoHome = new File(userHome, "auth-info");
	819	+	Commandline anotherGit = new Commandline("git");
	820	+	anotherGit.environments().put("HOME", authInfoHome.getAbsolutePath());
	821	+	cloneInfo.writeAuthData(authInfoHome, anotherGit, infoLogger, errorLogger);
783	822
784	823		File trustCertsHome = getTrustCertsHome();
785	824		if (trustCertsHome.exists()) {
		skipped 6 lines
792	831		infoLogger, errorLogger);
793	832		}
794	833
795		-	cloneRepository(workspace, cloneInfo.getCloneUrl(), commitHash, cloneDepth, git, infoLogger, errorLogger);
	834	+	cloneRepository(git, cloneInfo.getCloneUrl(), jobData.getCommitHash(), cloneDepth,
	835	+	infoLogger, errorLogger);
	836	+	addOriginRemote(git, cloneInfo.getCloneUrl(), infoLogger, errorLogger);
	837	+	updateSubmodulesIfNecessary(git, cloneDepth, infoLogger, errorLogger);
	838	+	}
796	839
	840	+	public static void addOriginRemote(Commandline git, String remoteUrl,
	841	+	LineConsumer infoLogger, LineConsumer errorLogger) {
797	842		git.clearArgs();
798		-	git.addArgs("remote", "add", "origin", cloneInfo.getCloneUrl());
	843	+	git.addArgs("remote", "add", "origin", remoteUrl);
799	844		git.execute(infoLogger, errorLogger).checkReturnCode();
	845	+	}
800	846
801		-	if (new File(workspace, ".gitmodules").exists()) {
802		-	logger.info("Retrieving submodules...");
	847	+	public static void runServerStep(String serverUrl, String jobToken, String positionStr,
	848	+	String encodedIncludeFiles, String encodedExcludeFiles, String encodedPlaceholders) {
	849	+	installJVMCert();
803	850
804		-	git.clearArgs();
805		-	git.addArgs("submodule", "update", "--init", "--recursive", "--force", "--quiet");
806		-	if (cloneDepth != 0)
807		-	git.addArgs("--depth=" + cloneDepth);
808		-	git.execute(infoLogger, new LineConsumer() {
	851	+	List<Integer> position = parsePosition(positionStr);
	852	+	Collection<String> includeFiles = decodeCommandArgAsCollection(encodedIncludeFiles);
	853	+	Collection<String> excludeFiles = decodeCommandArgAsCollection(encodedExcludeFiles);
	854	+	Collection<String> placeholders = decodeCommandArgAsCollection(encodedPlaceholders);
809	855
810		-	@Override
811		-	public void consume(String line) {
812		-	if (line.contains("Submodule") && line.contains("registered for path")
813		-	\|\| line.startsWith("From ") \|\| line.startsWith(" * branch")
814		-	\|\| line.startsWith(" +") && line.contains("->")) {
815		-	infoLogger.consume(line);
816		-	} else {
817		-	errorLogger.consume(line);
818		-	}
819		-	}
	856	+	TaskLogger logger = new TaskLogger() {
820	857
821		-	}).checkReturnCode();
822		-	}
	858	+	@Override
	859	+	public void log(String message, String taskId) {
	860	+	KubernetesHelper.logger.info(message);
	861	+	}
823	862
	863	+	};
	864	+	runServerStep(serverUrl, jobToken, position, includeFiles, excludeFiles, placeholders,
	865	+	getBuildHome(), getWorkspace(), logger);
824	866		}
825	867
826		-	public static void runServerStep(String serverUrl, String jobToken, String positionStr,
827		-	String encodedIncludeFiles, String encodedExcludeFiles, String encodedPlaceholders) {
828		-	installJVMCert();
829		-
	868	+	public static void runServerStep(String serverUrl, String jobToken, List<Integer> position,
	869	+	Collection<String> includeFiles, Collection<String> excludeFiles,
	870	+	Collection<String> placeholders, File buildHome, File workspace, TaskLogger logger) {
830	871		Client client = ClientBuilder.newClient();
831	872		client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");
832	873		try {
		skipped 1 lines
834	875		Invocation.Builder builder = target.request();
835	876		builder.header(HttpHeaders.AUTHORIZATION, BEARER + " " + jobToken);
836	877
837		-	List<Integer> position = parsePosition(positionStr);
838		-	Collection<String> includeFiles = decodeCommandArgAsCollection(encodedIncludeFiles);
839		-	Collection<String> excludeFiles = decodeCommandArgAsCollection(encodedExcludeFiles);
840		-	Collection<String> placeholders = decodeCommandArgAsCollection(encodedPlaceholders);
841		-
842		-	Map<String, String> placeholderValues = readPlaceholderValues(getBuildHome(), placeholders);
	878	+	Map<String, String> placeholderValues = readPlaceholderValues(buildHome, placeholders);
843	879
844	880		StreamingOutput os = new StreamingOutput() {
845	881
		skipped 10 lines
856	892		}
857	893
858	894		FileUtils.tar(
859		-	getWorkspace(),
	895	+	workspace,
860	896		replacePlaceholders(includeFiles, placeholderValues),
861	897		replacePlaceholders(excludeFiles, placeholderValues),
862	898		os, false);
		skipped 5 lines
868	904		checkStatus(response);
869	905		try (InputStream is = response.readEntity(InputStream.class)) {
870	906		while (readInt(is) == 1) {
871		-	logger.info(readString(is));
	907	+	logger.log(readString(is));
872	908		}
873	909		byte[] bytes = new byte[readInt(is)];
874	910		IOUtils.readFully(is, bytes);
		skipped 1 lines
876	912		for (Map.Entry<String, byte[]> entry: files.entrySet()) {
877	913		try {
878	914		FileUtils.writeByteArrayToFile(
879		-	new File(getBuildHome(), entry.getKey()),
	915	+	new File(buildHome, entry.getKey()),
880	916		entry.getValue());
881	917		} catch (IOException e) {
882	918		throw new RuntimeException(e);
		skipped 69 lines
952	988		for (String each: collection)
953	989		replacedCollection.add(replacePlaceholders(each, buildHome));
954	990		return replacedCollection;
955		-	}
956		-
957		-	public static String wrapWithAnsiError(String text) {
958		-	return "\033[1;31m" + text + "\033[0m";
959		-	}
960		-
961		-	public static String wrapWithAnsiWarning(String text) {
962		-	return "\033[1;33m" + text + "\033[0m";
963		-	}
964		-
965		-	public static String wrapWithAnsiEmphasize(String text) {
966		-	return "\033[1;35m" + text + "\033[0m";
967	991		}
968	992
969	993		}
		skipped 1 lines

■ ■ ■ ■ ■ ■

src/main/java/io/onedev/k8shelper/RunServerStep.java

		skipped 4 lines
5	5
6	6		import com.google.common.base.Preconditions;
7	7
	8	+	import io.onedev.commons.utils.TaskLogger;
	9	+
8	10		public class RunServerStep {
9	11
10	12		private static final Logger logger = LoggerFactory.getLogger(RunServerStep.class);
		skipped 10 lines
21	23
22	24		KubernetesHelper.runServerStep(serverUrl, jobToken, args[0], args[1], args[2], args[3]);
23	25		} catch (Exception e) {
24		-	logger.error(KubernetesHelper.wrapWithAnsiError("Error executing step"), e);
	26	+	logger.error(TaskLogger.wrapWithAnsiError("Error executing step"), e);
25	27		exitCode = 1;
26	28		} finally {
27	29		logger.info(KubernetesHelper.LOG_END_MESSAGE);
		skipped 6 lines

■ ■ ■ ■ ■ ■

src/main/java/io/onedev/k8shelper/SideCar.java

		skipped 4 lines
5	5
6	6		import com.google.common.base.Preconditions;
7	7
	8	+	import io.onedev.commons.utils.TaskLogger;
	9	+
8	10		public class SideCar {
9	11
10	12		private static final Logger logger = LoggerFactory.getLogger(SideCar.class);
		skipped 12 lines
23	25		logger.info(KubernetesHelper.LOG_END_MESSAGE);
24	26		System.exit(0);
25	27		} catch (Exception e) {
26		-	logger.error(KubernetesHelper.wrapWithAnsiError("Error executing sidecar logic"), e);
	28	+	logger.error(TaskLogger.wrapWithAnsiError("Error executing sidecar logic"), e);
27	29		logger.info(KubernetesHelper.LOG_END_MESSAGE);
28	30		System.exit(1);
29	31		}
		skipped 4 lines

Refactor to pull out some common functionality