build.c: -j, exact header deps, uses the compiler that built it

1 files changed, 240 insertions, 79 deletions

diff --git a/build.c b/build.c
index 18c12d8..9938016 100644
--- a/build.c
+++ b/build.c
@@ -8,6 +8,10 @@
  * it rebuilds itself when this file or kon/konbuild.h changes
  */
 
+#ifdef _MSC_VER
+#error "build.c uses gcc and clang flags, on Windows build it with mingw or llvm-mingw"
+#endif
+
 #define _POSIX_C_SOURCE 200809L
 #define KONBUILD_IMPLEMENTATION
 #include "kon/konbuild.h"
@@ -110,7 +114,7 @@ static void usage(void)
 {
 	size_t i;
 
-	fprintf(stderr, "usage: build [-d] [-D] [-p platform] [-m module,...] [-o objdir] [target ...]\n");
+	fprintf(stderr, "usage: build [-d] [-D] [-v] [-j jobs] [-p platform] [-m module,...] [-o objdir] [target ...]\n");
 	fprintf(stderr, "modules:  ");
 	for (i = 0; i < KNOWN; i++) fprintf(stderr, " %s", known[i].name);
 	fprintf(stderr, "\nplatforms: native linux bsd windows\n");
@@ -148,6 +152,13 @@ static void pushWords(KonCmd *to, const char *words)
 	while ((w = word(&copy, ' ')) != NULL) push(to, w);
 }
 
+/* CC can be a program with arguments, like "ccache cc". one that is a file on its own is taken whole, a path on Windows has spaces in it */
+static void pushCompiler(KonCmd *to)
+{
+	if (kon_exists(compiler)) push(to, compiler);
+	else pushWords(to, compiler);
+}
+
 static char *upper(const char *s)
 {
 	char *u = kon_fmt("%s", s), *p;
@@ -170,6 +181,13 @@ static char *exePath(const char *name)
 
 /*** compiling ***/
 
+/* gcc and clang write a file of the headers a source read when asked to, the others have none of it and rebuild when any header changes */
+#if defined(__TINYC__)
+#define HAVE_DEPS 0
+#else
+#define HAVE_DEPS 1
+#endif
+
 /* the warnings the engine's own code is held to. the stb files and miniaudio are not ours, they get -w */
 static void ourFlags(KonCmd *c, int withDefs)
 {
@@ -192,6 +210,7 @@ static void extFlags(KonCmd *c)
 	push(c, "-I" EXTDIR);
 }
 
+/* for what has no dependency file: older than a file it is made from, or than any header of the engine */
 static int stale(const char *output, const char **inputs, size_t count, int headers)
 {
 	long long m = kon_mtime(output);
@@ -201,51 +220,145 @@ static int stale(const char *output, const char **inputs, size_t count, int head
 	return kon_needsRebuild(output, inputs, count);
 }
 
-static void run(KonCmd *c)
+/* obj/name.o has its dependencies in obj/name.d */
+static char *depPath(const char *object)
+{
+	char *d = kon_fmt("%s", object);
+	size_t n = strlen(d);
+
+	if (n > 2 && d[n - 2] == '.') d[n - 1] = 'd';
+	return d;
+}
+
+/*** jobs ***/
+
+#define JOBS_MAX 256
+#define JOBS_LIMIT 32
+
+typedef struct job
+{
+	KonCmd cmd;
+	const char *label;
+} job_t;
+
+static job_t jobs[JOBS_MAX];
+static int jobCount;
+static int maxJobs = 1;
+
+static void queue(const char *label, KonCmd *c)
+{
+	if (jobCount >= JOBS_MAX) {
+		fprintf(stderr, "build: too many jobs\n");
+		exit(1);
+	}
+	jobs[jobCount].cmd = *c;
+	jobs[jobCount].label = label;
+	jobCount++;
+	*c = (KonCmd){0};
+}
+
+static void showLog(const char *path)
 {
-	if (!kon_runCmdSync(c)) exit(1);
-	kon_freeCmd(c);
+	FILE *f = fopen(path, "rb");
+	char buf[4096];
+	size_t n;
+
+	if (!f) return;
+	while ((n = fread(buf, 1, sizeof(buf), f)) > 0) fwrite(buf, 1, n, stderr);
+	fclose(f);
 }
 
-/* returns 1 when it compiled something */
-static int compile(const char *source, const char *object, int ext, const KonCmd *extra)
+/* runs what is queued, maxJobs at a time. what each one prints is kept in a file and shown when it is done, so the messages of two
+   compilers do not run into each other. after a failure nothing new starts, the ones running finish, and the build stops */
+static void runJobs(void)
+{
+	KonProc procs[JOBS_LIMIT];
+	int running = 0, failed = 0, next = 0, i;
+	int limit = maxJobs < 1 ? 1 : (maxJobs > JOBS_LIMIT ? JOBS_LIMIT : maxJobs);
+
+	for (i = 0; i < JOBS_LIMIT; i++) procs[i].active = 0;
+	if (jobCount > 0) kon_mkdirs(objdir);
+	while (next < jobCount || running > 0) {
+		while (!failed && next < jobCount && running < limit) {
+			const char *log;
+			int s = 0;
+
+			while (procs[s].active) s++;
+			log = kon_fmt("%s/.job%d.log", objdir, s);
+			kon_remove(log);
+			printf("%s\n", jobs[next].label);
+			if (!kon_startCmd(&jobs[next].cmd, log, &procs[s])) {
+				failed = 1;
+				break;
+			}
+			next++;
+			running++;
+		}
+		if (running == 0) break;
+		{
+			bool ok = false;
+			int s = kon_waitAny(procs, (size_t)limit, &ok);
+			const char *log;
+
+			if (s < 0) {
+				failed = 1;
+				break;
+			}
+			running--;
+			log = kon_fmt("%s/.job%d.log", objdir, s);
+			showLog(log);
+			kon_remove(log);
+			if (!ok) failed = 1;
+		}
+	}
+	for (i = 0; i < jobCount; i++) kon_freeCmd(&jobs[i].cmd);
+	jobCount = 0;
+	if (failed) exit(1);
+}
+
+static void queueCompile(const char *source, const char *object, int ext, const KonCmd *extra)
 {
 	KonCmd c = {0};
-	const char *inputs[1];
+	const char *dep = depPath(object);
 
-	inputs[0] = source;
-	if (!stale(object, inputs, 1, 1)) return 0;
-	kon_mkdirs(objdir);
-	printf("#   compile  %s\n", object);
-	fflush(stdout);
-	push(&c, compiler);
+#if HAVE_DEPS
+	if (!kon_depsNewer(object, dep)) return;
+#else
+	{
+		const char *in[1];
+
+		in[0] = source;
+		if (!stale(object, in, 1, 1)) return;
+	}
+#endif
+	pushCompiler(&c);
 	if (ext) extFlags(&c);
 	else ourFlags(&c, 1);
 	if (extra) pushAll(&c, extra);
+#if HAVE_DEPS
+	push(&c, "-MMD");
+	push(&c, "-MF");
+	push(&c, dep);
+#endif
 	push(&c, "-c");
 	push(&c, source);
 	push(&c, "-o");
 	push(&c, object);
-	run(&c);
-	return 1;
+	queue(kon_fmt("#   compile  %s", object), &c);
 }
 
-static int linkProgram(const char *output, const char **objects, size_t count, const KonCmd *flags, int withLibs)
+static void queueLink(const char *output, const char **objects, size_t count, int withLibs)
 {
 	KonCmd c = {0};
 	size_t i;
 
-	if (!stale(output, objects, count, 0)) return 0;
-	printf("#   link     %s\n", output);
-	fflush(stdout);
-	push(&c, compiler);
+	if (!stale(output, objects, count, 0)) return;
+	pushCompiler(&c);
 	push(&c, "-o");
 	push(&c, output);
-	if (flags) pushAll(&c, flags);
 	for (i = 0; i < count; i++) push(&c, objects[i]);
 	if (withLibs) pushAll(&c, &libs);
-	run(&c);
-	return 1;
+	queue(kon_fmt("#   link     %s", output), &c);
 }
 
 /*** what goes into the program ***/
@@ -356,8 +469,9 @@ static void setup(void)
 
 	compiler = getenv("CC");
 	if (!compiler || !*compiler) {
-		if (strcmp(platform, "windows") == 0) compiler = native ? "gcc" : "x86_64-w64-mingw32-gcc";
-		else compiler = "cc";
+		/* the one that built this program, see konbuild.h. a Windows build from another system has to be a cross compiler */
+		if (strcmp(platform, "windows") == 0 && !native) compiler = "x86_64-w64-mingw32-gcc";
+		else compiler = kon_cc();
 	}
 
 	if (debug) {
@@ -395,9 +509,10 @@ static void setup(void)
 	for (i = 0; i < chosenCount; i++) addModuleUnits(chosen[i]);
 
 	{
-		long long a = kon_newestIn(KONDIR, ".h"), b = kon_newestIn(EXTDIR, ".h");
+		long long a = kon_newestIn(KONDIR, ".h"), b = kon_newestIn(EXTDIR, ".h"), t = kon_newestIn("tools", ".h");
 
 		newestHeader = a > b ? a : b;
+		if (t > newestHeader) newestHeader = t;
 	}
 }
 
@@ -410,7 +525,7 @@ static void checkFlags(void)
 	FILE *f;
 	char old[8192];
 
-	push(&all, compiler);
+	pushCompiler(&all);
 	pushAll(&all, &modeFlags);
 	pushAll(&all, &defs);
 	pushAll(&all, &libs);
@@ -442,7 +557,7 @@ static void checkFlags(void)
 			for (i = 0; i < list.count; i++) {
 				size_t n = strlen(list.items[i]);
 
-				if (n > 2 && strcmp(list.items[i] + n - 2, ".o") == 0) kon_remove(list.items[i]);
+				if (n > 2 && (strcmp(list.items[i] + n - 2, ".o") == 0 || strcmp(list.items[i] + n - 2, ".d") == 0)) kon_remove(list.items[i]);
 			}
 			kon_freeList(&list);
 		}
@@ -460,7 +575,7 @@ static void checkFlags(void)
 static const char *objects[OBJECTS_MAX + 16];
 static int objectCount;
 
-static void buildObjects(int withEngine)
+static void queueObjects(int withEngine)
 {
 	int i;
 
@@ -469,15 +584,17 @@ static void buildObjects(int withEngine)
 		const char *obj = objPath(units[i].name);
 
 		if (!withEngine && strcmp(units[i].name, "engine") == 0) continue;
-		compile(units[i].source, obj, units[i].ext, NULL);
+		queueCompile(units[i].source, obj, units[i].ext, NULL);
 		objects[objectCount++] = obj;
 	}
 }
 
 static void targetAll(void)
 {
-	buildObjects(1);
-	linkProgram(exePath(PROGRAM), objects, (size_t)objectCount, NULL, 1);
+	queueObjects(1);
+	runJobs();
+	queueLink(exePath(PROGRAM), objects, (size_t)objectCount, 1);
+	runJobs();
 }
 
 static void targetRun(void)
@@ -490,45 +607,49 @@ static void targetRun(void)
 	kon_freeCmd(&c);
 }
 
+static const char *toolNames[] = {"konpack", "konmodel", "mzsurvey", "mzscript", "kons2c", "mzmap", "mzdata"};
+
 static const char *examples[] = {"modelview", "sound", "ui", "platform", "dialog", "script", "corridor", "hills", "sectorview"};
 
-static void targetExamples(void)
+/* only is the name of one example, or NULL for all of them */
+static void targetExamples(const char *only)
 {
 	size_t i;
 
-	buildObjects(0);
+	queueObjects(0);
+	for (i = 0; i < sizeof(examples) / sizeof(examples[0]); i++) {
+		if (only && strcmp(only, examples[i]) != 0) continue;
+		queueCompile(kon_fmt("examples/%s.c", examples[i]), objPath(examples[i]), 0, NULL);
+	}
+	runJobs();
 	for (i = 0; i < sizeof(examples) / sizeof(examples[0]); i++) {
-		const char *obj = objPath(examples[i]);
-		const char *src = kon_fmt("examples/%s.c", examples[i]);
 		const char *all[OBJECTS_MAX + 16];
 		int n = 0, k;
 
-		compile(src, obj, 0, NULL);
-		all[n++] = obj;
+		if (only && strcmp(only, examples[i]) != 0) continue;
+		all[n++] = objPath(examples[i]);
 		for (k = 0; k < objectCount; k++) all[n++] = objects[k];
-		linkProgram(exePath(examples[i]), all, (size_t)n, NULL, 1);
+		queueLink(exePath(examples[i]), all, (size_t)n, 1);
 	}
+	runJobs();
 }
 
 /* a program made straight from some files, with the headers they read checked too. the converters do not link the engine */
-static void tool(const char *name, const char **sources, size_t count, const KonCmd *flags, int withEngineFlags, int withMath)
+static void queueTool(const char *name, const char **sources, size_t count, const KonCmd *flags, int withEngineFlags, int withMath)
 {
 	KonCmd c = {0};
 	const char *out = exePath(name);
 	size_t i;
 
-	if (!stale(out, sources, count, 1) && kon_newestIn("tools", ".h") <= kon_mtime(out)) return;
-	kon_mkdirs(objdir);
-	printf("#   link     %s\n", out);
-	fflush(stdout);
-	push(&c, compiler);
+	if (!stale(out, sources, count, 1)) return;
+	pushCompiler(&c);
 	ourFlags(&c, withEngineFlags);
 	if (flags) pushAll(&c, flags);
 	push(&c, "-o");
 	push(&c, out);
 	for (i = 0; i < count; i++) push(&c, sources[i]);
 	if (withMath) push(&c, "-lm");
-	run(&c);
+	queue(kon_fmt("#   link     %s", out), &c);
 }
 
 static void targetTools(void)
@@ -536,24 +657,21 @@ static void targetTools(void)
 	KonCmd extra = {0};
 	const char *cgltfObj = objPath("cgltf");
 	const char *s[8];
-	KonCmd cg = {0};
 
 	/* the packer uses the module flags, the others are built without them */
 	push(&extra, "-DKONPACK_WRITER");
 	s[0] = "tools/konpack.c";
 	s[1] = KONDIR "/konpack.c";
-	tool("konpack", s, 2, &extra, 1, 0);
+	queueTool("konpack", s, 2, &extra, 1, 0);
 	kon_freeCmd(&extra);
 
 	{
+		KonCmd cg = {0};
 		const char *in[1];
 
 		in[0] = "tools/cgltf.c";
 		if (stale(cgltfObj, in, 1, 0)) {
-			printf("#   compile  %s\n", cgltfObj);
-			fflush(stdout);
-			kon_mkdirs(objdir);
-			push(&cg, compiler);
+			pushCompiler(&cg);
 			push(&cg, "-std=gnu99");
 			push(&cg, "-w");
 			pushAll(&cg, &modeFlags);
@@ -562,37 +680,41 @@ static void targetTools(void)
 			push(&cg, "tools/cgltf.c");
 			push(&cg, "-o");
 			push(&cg, cgltfObj);
-			run(&cg);
+			queue(kon_fmt("#   compile  %s", cgltfObj), &cg);
 		}
 	}
-	push(&extra, "-DKONSOFREN_3D");
-	push(&extra, "-DKONMODEL_WRITER");
-	s[0] = "tools/konmodel.c";
-	s[1] = KONDIR "/konmodel.c";
-	s[2] = KONDIR "/konassets.c";
-	s[3] = KONDIR "/konpack.c";
-	s[4] = SRCDIR "/impl/konsofren.c";
-	s[5] = cgltfObj;
-	tool("konmodel", s, 6, &extra, 0, 1);
-	kon_freeCmd(&extra);
 
 	s[0] = "tools/mzsurvey.c";
-	tool("mzsurvey", s, 1, NULL, 0, 0);
+	queueTool("mzsurvey", s, 1, NULL, 0, 0);
 
 	push(&extra, "-Itools");
 	s[0] = "tools/mzscript.c";
 	s[1] = KONDIR "/konscript.c";
-	tool("mzscript", s, 2, &extra, 0, 0);
+	queueTool("mzscript", s, 2, &extra, 0, 0);
 	s[0] = "tools/kons2c.c";
-	tool("kons2c", s, 2, &extra, 0, 0);
+	queueTool("kons2c", s, 2, &extra, 0, 0);
 	kon_freeCmd(&extra);
 
 	s[0] = "tools/mzmap.c";
-	tool("mzmap", s, 1, NULL, 0, 0);
+	queueTool("mzmap", s, 1, NULL, 0, 0);
 
 	s[0] = "tools/mzdata.c";
 	s[1] = KONDIR "/konbattle.c";
-	tool("mzdata", s, 2, NULL, 0, 1);
+	queueTool("mzdata", s, 2, NULL, 0, 1);
+
+	/* the model converter links the object the first batch makes */
+	runJobs();
+	push(&extra, "-DKONSOFREN_3D");
+	push(&extra, "-DKONMODEL_WRITER");
+	s[0] = "tools/konmodel.c";
+	s[1] = KONDIR "/konmodel.c";
+	s[2] = KONDIR "/konassets.c";
+	s[3] = KONDIR "/konpack.c";
+	s[4] = SRCDIR "/impl/konsofren.c";
+	s[5] = cgltfObj;
+	queueTool("konmodel", s, 6, &extra, 0, 1);
+	kon_freeCmd(&extra);
+	runJobs();
 }
 
 static void targetPack(void)
@@ -628,7 +750,8 @@ static void targetPack(void)
 	push(&c, assets);
 	push(&c, "-l");
 	push(&c, listFile);
-	run(&c);
+	if (!kon_runCmdSync(&c)) exit(1);
+	kon_freeCmd(&c);
 }
 
 static void removeFile(const char *path)
@@ -639,22 +762,26 @@ static void removeFile(const char *path)
 	}
 }
 
+static int endsWith(const char *s, const char *tail)
+{
+	size_t n = strlen(s), t = strlen(tail);
+
+	return n >= t && strcmp(s + n - t, tail) == 0;
+}
+
 static void targetClean(void)
 {
-	static const char *tools[] = {"konpack", "konmodel", "mzsurvey", "mzscript", "kons2c", "mzmap", "mzdata"};
 	KonList list = {0};
 	size_t i;
 
 	if (kon_listFiles(objdir, &list)) {
 		for (i = 0; i < list.count; i++) {
-			size_t n = strlen(list.items[i]);
-
-			if (n > 2 && strcmp(list.items[i] + n - 2, ".o") == 0) kon_remove(list.items[i]);
+			if (endsWith(list.items[i], ".o") || endsWith(list.items[i], ".d") || endsWith(list.items[i], ".log")) kon_remove(list.items[i]);
 		}
 		kon_freeList(&list);
 	}
 	removeFile(exePath(PROGRAM));
-	for (i = 0; i < sizeof(tools) / sizeof(tools[0]); i++) removeFile(exePath(tools[i]));
+	for (i = 0; i < sizeof(toolNames) / sizeof(toolNames[0]); i++) removeFile(exePath(toolNames[i]));
 	for (i = 0; i < sizeof(examples) / sizeof(examples[0]); i++) removeFile(exePath(examples[i]));
 	removeFile(kon_fmt("%s/data.kpk", objdir));
 	removeFile(kon_fmt("%s/assets.lst", objdir));
@@ -662,6 +789,35 @@ static void targetClean(void)
 
 /*** main ***/
 
+/* a target that is a program in the object folder, like obj/sound, as make takes them. returns 0 when it is none */
+static int targetFile(const char *t)
+{
+	size_t d = strlen(objdir), n, e = strlen(exe), i;
+	char *name;
+
+	if (strncmp(t, objdir, d) != 0 || t[d] != '/') return 0;
+	name = kon_fmt("%s", t + d + 1);
+	n = strlen(name);
+	if (e > 0 && n > e && strcmp(name + n - e, exe) == 0) name[n - e] = 0;
+	for (i = 0; i < sizeof(examples) / sizeof(examples[0]); i++) {
+		if (strcmp(name, examples[i]) == 0) {
+			targetExamples(name);
+			return 1;
+		}
+	}
+	for (i = 0; i < sizeof(toolNames) / sizeof(toolNames[0]); i++) {
+		if (strcmp(name, toolNames[i]) == 0) {
+			targetTools();
+			return 1;
+		}
+	}
+	if (strcmp(name, PROGRAM) == 0) {
+		targetAll();
+		return 1;
+	}
+	return 0;
+}
+
 static void addModuleList(const char *list)
 {
 	char *copy = kon_fmt("%s", list), *w;
@@ -672,10 +828,11 @@ static void addModuleList(const char *list)
 int main(int argc, char **argv)
 {
 	const char *targets[16];
-	int targetCount = 0, i;
+	int targetCount = 0, i, jobsAsked = 0;
 
 	KON_REBUILDSELF(argc, argv);
 	setvbuf(stdout, NULL, _IOLBF, 0);
+	kon_verbose = 0;
 
 	for (i = 1; i < argc; i++) {
 		const char *a = argv[i];
@@ -693,11 +850,14 @@ int main(int argc, char **argv)
 					debug = 1;
 				} else if (*p == 'D') {
 					developer = 1;
-				} else if (*p == 'p' || *p == 'm' || *p == 'o') {
+				} else if (*p == 'v') {
+					kon_verbose = 1;
+				} else if (*p == 'p' || *p == 'm' || *p == 'o' || *p == 'j') {
 					const char *value = p[1] ? p + 1 : (i + 1 < argc ? argv[++i] : NULL);
 
 					if (!value) usage();
 					if (*p == 'p') platform = value;
+					else if (*p == 'j') jobsAsked = atoi(value);
 					else if (*p == 'o') objdir = value;
 					else addModuleList(value);
 					break;
@@ -709,9 +869,10 @@ int main(int argc, char **argv)
 	}
 	if (targetCount == 0) targets[targetCount++] = "all";
 
+	maxJobs = jobsAsked > 0 ? jobsAsked : kon_cpuCount();
 	setup();
 	if (hasModule("debug") && !(debug || developer)) printf("note: debug module does nothing without -d or -D\n");
-	printf("===> platform %s, %s build, modules:", platform, debug ? "debug" : "release");
+	printf("===> platform %s, %s build, compiler %s, modules:", platform, debug ? "debug" : "release", compiler);
 	if (chosenCount == 0) printf(" none");
 	for (i = 0; i < chosenCount; i++) printf(" %s", chosen[i]);
 	printf("\n");
@@ -725,8 +886,8 @@ int main(int argc, char **argv)
 		else if (strcmp(t, "clean") == 0) targetClean();
 		else if (strcmp(t, "tools") == 0) targetTools();
 		else if (strcmp(t, "pack") == 0) targetPack();
-		else if (strcmp(t, "examples") == 0) targetExamples();
-		else {
+		else if (strcmp(t, "examples") == 0) targetExamples(NULL);
+		else if (!targetFile(t)) {
 			fprintf(stderr, "build: unknown target '%s'\n", t);
 			usage();
 		}