initial modifications for ccache to support SWIG

git-svn-id: https://swig.svn.sourceforge.net/svnroot/swig/trunk@10915 626c5289-ae23-0410-ae9c-e8d60b6d4f22

CCache/README.swig

@@ -0,0 +1,3 @@
+This directory contains a version of ccache. The initial version was based on ccache-2.4 plus 
+debian patches 01-02, 04-12, see the debian/patches subdirectory. Changes have been made to
+support cacheing the output from SWIG. The ability to cache c/c++ compiler output has been retained.

CCache/ccache.c

@@ -23,6 +23,9 @@
 
 #include "ccache.h"
 
+/* verbose mode */
+int ccache_verbose = 0;
+
 /* the base cache directory */
 char *cache_dir = NULL;
 
@@ -65,6 +68,9 @@ char *stats_file = NULL;
 /* can we safely use the unification hashing backend? */
 static int enable_unify;
 
+/* customisation for using the SWIG compiler */
+static int swig;
+
 /* a list of supported file extensions, and the equivalent
    extension for code that has been through the pre-processor
 */
@@ -122,6 +128,14 @@ static void failed(void)
 		args_add_prefix(orig_args, p);
 	}
 
+	if (ccache_verbose) {
+		display_execute_args(orig_args->argv);
+	}
+
+	if (swig) {
+		putenv("CCACHE_OUTFILES");
+	}
+
 	execv(orig_args->argv[0], orig_args->argv);
 	cc_log("execv returned (%s)!\n", strerror(errno));
 	perror(orig_args->argv[0]);
@@ -149,21 +163,43 @@ static const char *tmp_string(void)
 	return ret;
 }
 
+/* update cached file sizes and count helper function for to_cache() */
+static void to_cache_stats_helper(struct stat *pstat, char *cached_filename, char *tmp_outfiles, int *files_size, int *cached_files_count)
+{
+#if ENABLE_ZLIB
+	/* do an extra stat on the cache file for the size statistics */
+	if (stat(cached_filename, pstat) != 0) {
+		cc_log("failed to stat cache files - %s\n", strerror(errno));
+		stats_update(STATS_ERROR);
+		if (tmp_outfiles) {
+			unlink(tmp_outfiles);
+		}
+		failed();
+	}
+#endif
+	(*files_size) += file_size(pstat);
+	(*cached_files_count)++;
+}
 
 /* run the real compiler and put the result in cache */
 static void to_cache(ARGS *args)
 {
 	char *path_stderr;
-	char *tmp_stdout, *tmp_stderr, *tmp_hashname;
+	char *tmp_stdout, *tmp_stderr, *tmp_outfiles;
-	struct stat st1, st2;
+	struct stat st1;
 	int status;
+	int cached_files_count = 0;
+	int files_size = 0;
 
 	x_asprintf(&tmp_stdout, "%s/tmp.stdout.%s", temp_dir, tmp_string());
 	x_asprintf(&tmp_stderr, "%s/tmp.stderr.%s", temp_dir, tmp_string());
-	x_asprintf(&tmp_hashname, "%s/tmp.hash.%s.o", temp_dir, tmp_string());
+	x_asprintf(&tmp_outfiles, "%s/tmp.outfiles.%s", temp_dir, tmp_string());
 
+	/* TODO swig */
+	if (output_file) {
 		args_add(args, "-o");
-	args_add(args, tmp_hashname);
+		args_add(args, output_file);
+	}
 
 	/* Turn off DEPENDENCIES_OUTPUT when running cc1, because
 	 * otherwise it will emit a line like
@@ -173,33 +209,46 @@ static void to_cache(ARGS *args)
 	 * unsetenv() is on BSD and Linux but not portable. */
 	putenv("DEPENDENCIES_OUTPUT");
 
+	/* Give SWIG a filename for it to create and populate with a list of files that it generates */
+	if (swig) {
+		char *ccache_outfiles;
+		x_asprintf(&ccache_outfiles, "CCACHE_OUTFILES=%s", tmp_outfiles);
+		unlink(tmp_outfiles);
+		if (getenv("CCACHE_OUTFILES") || putenv(ccache_outfiles) == -1) {
+			stats_update(STATS_ERROR);
+			failed();
+		}
+	}
+	
 	if (getenv("CCACHE_CPP2")) {
 		args_add(args, input_file);
 	} else {
+		if (swig) {
+			args_add(args, "-nopreprocess");
+		}
 		args_add(args, i_tmpfile);
 	}
 	status = execute(args->argv, tmp_stdout, tmp_stderr);
 	args_pop(args, 3);
 
 	if (stat(tmp_stdout, &st1) != 0 || st1.st_size != 0) {
-		cc_log("compiler produced stdout for %s\n", output_file);
+		cc_log("compiler produced stdout for %s\n", input_file);
 		stats_update(STATS_STDOUT);
 		unlink(tmp_stdout);
 		unlink(tmp_stderr);
-		unlink(tmp_hashname);
+		unlink(tmp_outfiles);
+		if (!swig) unlink(output_file);
 		failed();
 	}
 	unlink(tmp_stdout);
 
 	if (status != 0) {
 		int fd;
-		cc_log("compile of %s gave status = %d\n", output_file, status);
+		cc_log("compile of %s gave status = %d\n", input_file, status);
 		stats_update(STATS_STATUS);
 
 		fd = open(tmp_stderr, O_RDONLY | O_BINARY);
 		if (fd != -1) {
-			if (strcmp(output_file, "/dev/null") == 0 ||
-			    move_file(tmp_hashname, output_file) == 0 || errno == ENOENT) {
 			if (cpp_stderr) {
 				/* we might have some stderr from cpp */
 				int fd2 = open(cpp_stderr, O_RDONLY | O_BINARY);
@@ -221,41 +270,93 @@ static void to_cache(ARGS *args)
 			}
 			exit(status);
 		}
-		}
 		
 		unlink(tmp_stderr);
-		unlink(tmp_hashname);
+		unlink(tmp_outfiles);
+		if (!swig) unlink(output_file);
 		failed();
+	} else {
+		int hardlink = (getenv("CCACHE_NOCOMPRESS") != 0) && (getenv("CCACHE_HARDLINK") != 0);
+		if (swig) {
+			/* read the list of generated files and copy each of them into the cache */
+			FILE *file;
+			file = fopen(tmp_outfiles, "r");
+			if (file) {
+				char out_filename[FILENAME_MAX + 1];
+				char out_filename_cache[FILENAME_MAX + 1];
+				while (fgets(out_filename, FILENAME_MAX, file)) {
+					char *linefeed = strchr(out_filename, '\n');
+					if (linefeed) {
+						*linefeed = 0;
+
+						if (cached_files_count == 0) {
+							strcpy(out_filename_cache, hashname);
+						} else {
+							sprintf(out_filename_cache, "%s.%d", hashname, cached_files_count);
+						}
+
+						if (stat(out_filename, &st1) != 0 ||
+								commit_to_cache(out_filename, out_filename_cache, hardlink) != 0) {
+							fclose(file);
+							unlink(tmp_outfiles);
+							failed();
+						}
+						to_cache_stats_helper(&st1, out_filename_cache, tmp_outfiles, &files_size, &cached_files_count);
+					} else {
+						cached_files_count = 0;
+						break;
+					}
+				}
+				fclose(file);
+				if (cached_files_count == 0) {
+					cc_log("failed to copy output files to cache - internal error\n");
+					stats_update(STATS_ERROR);
+					unlink(tmp_outfiles);
+					failed();
+				}
+
+				/* also copy the (uncompressed) file containing the list of generated files into the cache */
+				sprintf(out_filename_cache, "%s.outfiles", hashname);
+				if (stat(tmp_outfiles, &st1) != 0 ||
+					safe_rename(tmp_outfiles, out_filename_cache) != 0) {
+					cc_log("failed to copy outfiles file to cache - %s\n", strerror(errno));
+					stats_update(STATS_ERROR);
+					unlink(tmp_outfiles);
+					failed();
+				}
+				to_cache_stats_helper(&st1, out_filename_cache, tmp_outfiles, &files_size, &cached_files_count);
+				unlink(tmp_outfiles);
+			} else {
+				cc_log("failed to open temp outfiles file - %s\n", strerror(errno));
+				stats_update(STATS_ERROR);
+				failed();
+			}
+		} else {
+			if (stat(output_file, &st1) != 0 ||
+				commit_to_cache(output_file, hashname, hardlink) != 0) {
+				failed();
+			}
+			to_cache_stats_helper(&st1, hashname, 0, &files_size, &cached_files_count);
+		}
 	}
 
 	x_asprintf(&path_stderr, "%s.stderr", hashname);
 
 	if (stat(tmp_stderr, &st1) != 0 ||
-		stat(tmp_hashname, &st2) != 0 ||
-		move_file(tmp_hashname, hashname) != 0 ||
 		move_file(tmp_stderr, path_stderr) != 0) {
 		cc_log("failed to rename tmp files - %s\n", strerror(errno));
 		stats_update(STATS_ERROR);
 		failed();
 	}
 
-#if ENABLE_ZLIB
+	to_cache_stats_helper(&st1, path_stderr, 0, &files_size, &cached_files_count);
-	/* do an extra stat on the cache files for
-	   the size statistics */
-	if (stat(path_stderr, &st1) != 0 ||
-		stat(hashname, &st2) != 0) {
-		cc_log("failed to stat cache files - %s\n", strerror(errno));
-		stats_update(STATS_ERROR);
-		failed();
-    }
-#endif
 
-	cc_log("Placed %s into cache\n", output_file);
+	cc_log("Placed %d files for %s into cache\n", cached_files_count, input_file);
-	stats_tocache(file_size(&st1) + file_size(&st2));
+	stats_tocache(files_size, cached_files_count);
 
-	free(tmp_hashname);
 	free(tmp_stderr);
 	free(tmp_stdout);
+	free(tmp_outfiles);
 	free(path_stderr);
 }
 
@@ -288,7 +389,7 @@ static void find_hash(ARGS *args)
 	}
 
 	/* we have to hash the extension, as a .i file isn't treated the same
-	   by the compiler as a .ii file */
+	   by the compiler as a .ii file (Note: not strictly necessary for SWIG) */
 	hash_string(i_extension);
 
 	/* first the arguments */
@@ -383,7 +484,7 @@ static void find_hash(ARGS *args)
 		/* we are compiling a .i or .ii file - that means we
 		   can skip the cpp stage and directly form the
 		   correct i_tmpfile */
-		path_stdout = input_file;
+		path_stdout = x_strdup(input_file);
 		if (create_empty_file(path_stderr) != 0) {
 			stats_update(STATS_ERROR);
 			cc_log("failed to create empty stderr file\n");
@@ -456,7 +557,6 @@ static void find_hash(ARGS *args)
 	free(hash_dir);
 }
 
-
 /* 
    try to return the compile result from cache. If we can return from
    cache then this function exits with the correct status code,
@@ -465,7 +565,6 @@ static void from_cache(int first)
 {
 	int fd_stderr, fd_cpp_stderr;
 	char *stderr_file;
-	int ret;
 	struct stat st;
 
 	x_asprintf(&stderr_file, "%s.stderr", hashname);
@@ -488,57 +587,82 @@ static void from_cache(int first)
 #ifndef ENABLE_ZLIB
 	/* if the cache file is compressed we must recache */
 	if ((first && getenv("CCACHE_RECACHE")) ||
-		test_if_compressed(hashname) == 1) {
+		test_if_compressed(hashname) == 1)
 #else
-	if (first && getenv("CCACHE_RECACHE")) {
+	if (first && getenv("CCACHE_RECACHE"))
 #endif
+	{
 		close(fd_stderr);
 		unlink(stderr_file);
 		free(stderr_file);
 		return;
 	}
 
+	if (first) {
+		int hardlink;
+		int passfail = -1;
+
 		/* update timestamps for LRU cleanup
 		   also gives output_file a sensible mtime when hard-linking (for make) */
-#ifdef HAVE_UTIMES
+		x_utimes(stderr_file);
-       utimes(hashname, NULL);
-	utimes(stderr_file, NULL);
-#else
-	utime(hashname, NULL);
-	utime(stderr_file, NULL);
-#endif
 
-	if (strcmp(output_file, "/dev/null") == 0) {
+		hardlink = (getenv("CCACHE_HARDLINK") != 0);
-		ret = 0;
+
+		if (swig) {
+			/* read the list of generated files and copy each of them out of the cache */
+			FILE *file;
+			char *outfiles;
+			x_asprintf(&outfiles, "%s.outfiles", hashname);
+			file = fopen(outfiles, "r");
+			if (file) {
+				char out_filename[FILENAME_MAX + 1];
+				char out_filename_cache[FILENAME_MAX + 1];
+				int retrieved_files_count = 0;
+				x_utimes(outfiles);
+				while (fgets(out_filename, FILENAME_MAX, file)) {
+					char *linefeed = strchr(out_filename, '\n');
+					if (linefeed) {
+						*linefeed = 0;
+
+						if (retrieved_files_count == 0) {
+							strcpy(out_filename_cache, hashname);
 						} else {
-		unlink(output_file);
+							sprintf(out_filename_cache, "%s.%d", hashname, retrieved_files_count);
-		/* only make a hardlink if the cache file is uncompressed */
-		if (getenv("CCACHE_HARDLINK") &&
-			test_if_compressed(hashname) == 0) {
-			ret = link(hashname, output_file);
-		} else {
-			ret = copy_file(hashname, output_file);
-		}
 						}
 
-	/* the hash file might have been deleted by some external process */
+						passfail = retrieve_from_cache(out_filename_cache, out_filename, hardlink);
-	if (ret == -1 && errno == ENOENT) {
+						if (passfail == -1) {
-		cc_log("hashfile missing for %s\n", output_file);
+							break;
-		stats_update(STATS_MISSING);
+						}
+
+						retrieved_files_count++;
+					} else {
+						cc_log("failed to copy output files from cache - internal error\n");
+						stats_update(STATS_ERROR);
+						passfail = -1;
+						break;
+					}
+				}
+				if (retrieved_files_count == 0) {
+					cc_log("failed to copy output files from cache - internal error\n");
+					stats_update(STATS_ERROR);
+					passfail = -1;
+				}
+				fclose(file);
+			} else {
+				cc_log("failed to open cached outfiles file - %s\n", strerror(errno));
+				stats_update(STATS_ERROR);
+			}
+		} else {
+			passfail = retrieve_from_cache(hashname, output_file, hardlink);
+		}
+
+		free(stderr_file);
+		if (passfail == -1) {
 			close(fd_stderr);
 			unlink(stderr_file);
 			return;
 		}
-	free(stderr_file);
-
-	if (ret == -1) {
-		ret = copy_file(hashname, output_file);
-		if (ret == -1) {
-			cc_log("failed to copy %s -> %s (%s)\n", 
-			       hashname, output_file, strerror(errno));
-			stats_update(STATS_ERROR);
-			failed();
-		}
 	}
 
 	/* get rid of the intermediate preprocessor file */
@@ -566,7 +690,7 @@ static void from_cache(int first)
 
 	/* and exit with the right status code */
 	if (first) {
-		cc_log("got cached result for %s\n", output_file);
+		cc_log("got cached result for %s\n", input_file);
 		stats_update(STATS_CACHED);
 	}
 
@@ -597,7 +721,7 @@ static void find_compiler(int argc, char **argv)
 
 	/* support user override of the compiler */
 	if ((path=getenv("CCACHE_CC"))) {
-		base = strdup(path);
+		base = x_strdup(path);
 	}
 
 	orig_args->argv[0] = find_executable(base, MYNAME);
@@ -622,6 +746,8 @@ static const char *check_extension(const char *fname, int *direct_i)
 		*direct_i = 0;
 	}
 
+	if (swig) return "ii"; /* any file extension is acceptable as input for SWIG */
+
 	p = strrchr(fname, '.');
 	if (!p) return NULL;
 	p++;
@@ -662,6 +788,11 @@ static void process_args(int argc, char **argv)
 
 	args_add(stripped_args, argv[0]);
 
+	/* -c not required for SWIG */
+	if (swig) {
+		found_c_opt = 1;
+	}
+
 	for (i=1; i<argc; i++) {
 		/* some options will never work ... */
 		if (strcmp(argv[i], "-E") == 0) {
@@ -709,10 +840,12 @@ static void process_args(int argc, char **argv)
 		}
 		
 		/* alternate form of -o, with no space */
+		if (!swig) { /* some of SWIG's arguments begin with -o */
 			if (strncmp(argv[i], "-o", 2) == 0) {
 				output_file = &argv[i][2];
 				continue;
 			}
+		}
 
 		/* debugging is handled specially, so that we know if we
 		   can strip line number info 
@@ -747,6 +880,13 @@ static void process_args(int argc, char **argv)
 			dependency_target_specified = 1;
 		}
 
+		/* the input file is already preprocessed */
+		if (swig && strcmp(argv[i], "-nopreprocess") == 0) {
+			args_add(stripped_args, argv[i]);
+			direct_i_file = 1;
+			continue;
+		}
+
 		/* options that take an argument */
 		{
 			const char *opts[] = {"-I", "-include", "-imacros", "-iprefix",
@@ -817,7 +957,11 @@ static void process_args(int argc, char **argv)
 		failed();
 	}
 
+	if (swig) {
+		i_extension = check_extension(input_file, NULL);
+	} else {
 		i_extension = check_extension(input_file, &direct_i_file);
+	}
 	if (i_extension == NULL) {
 		cc_log("Not a C/C++ file - %s\n", input_file);
 		stats_update(STATS_NOTC);
@@ -843,7 +987,7 @@ static void process_args(int argc, char **argv)
 		failed();
 	}
 
-	if (!output_file) {
+	if (!swig && !output_file) {
 		char *p;
 		output_file = x_strdup(input_file);
 		if ((p = strrchr(output_file, '/'))) {
@@ -895,7 +1039,7 @@ static void process_args(int argc, char **argv)
 	}
 
 	/* cope with -o /dev/null */
-	if (strcmp(output_file,"/dev/null") != 0 && stat(output_file, &st) == 0 && !S_ISREG(st.st_mode)) {
+	if (output_file && strcmp(output_file,"/dev/null") != 0 && stat(output_file, &st) == 0 && !S_ISREG(st.st_mode)) {
 		cc_log("Not a regular file %s\n", output_file);
 		stats_update(STATS_DEVICE);
 		failed();
@@ -934,6 +1078,10 @@ static void ccache(int argc, char *argv[])
 		enable_unify = 1;
 	}
 
+	if (getenv("CCACHE_SWIG")) {
+		swig = 1;
+	}
+
 	/* process argument list, returning a new set of arguments for pre-processing */
 	process_args(orig_args->argc, orig_args->argv);
 
@@ -963,7 +1111,7 @@ static void ccache(int argc, char *argv[])
 
 static void usage(void)
 {
-	printf("ccache, a compiler cache. Version %s\n", CCACHE_VERSION);
+	printf("ccache, a compiler cache, with modifications for SWIG. Version %s\n", CCACHE_VERSION);
 	printf("Copyright Andrew Tridgell, 2002\n\n");
 	
 	printf("Usage:\n");
@@ -998,7 +1146,7 @@ static int ccache_main(int argc, char *argv[])
 	while ((c = getopt(argc, argv, "hszcCF:M:V")) != -1) {
 		switch (c) {
 		case 'V':
-			printf("ccache version %s\n", CCACHE_VERSION);
+			printf("ccache with modifications for SWIG version %s\n", CCACHE_VERSION);
 			printf("Copyright Andrew Tridgell 2002\n");
 			printf("Released under the GNU GPL v2 or later\n");
 			exit(0);
@@ -1097,13 +1245,12 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	temp_dir = getenv("CCACHE_TEMPDIR");
-	if (!temp_dir) {
-		temp_dir = cache_dir;
-	}
-
 	cache_logfile = getenv("CCACHE_LOGFILE");
 
+	if (getenv("CCACHE_VERBOSE")) {
+		ccache_verbose = 1;
+	}
+
 	setup_uncached_err();
 	
 
@@ -1140,6 +1287,17 @@ int main(int argc, char *argv[])
 		exit(1);
 	}
 
+	temp_dir = getenv("CCACHE_TEMPDIR");
+	if (!temp_dir) {
+		x_asprintf(&temp_dir, "%s/temp", cache_dir);
+		/* make sure temp dir exists if not supplied by user */
+		if (temp_dir && create_dir(temp_dir) != 0) {
+			fprintf(stderr,"ccache: failed to create %s (%s)\n", 
+				temp_dir, strerror(errno));
+			exit(1);
+		}
+	}
+
 	if (!getenv("CCACHE_READONLY")) {
 		if (create_cachedirtag(cache_dir) != 0) {
 			fprintf(stderr,"ccache: failed to create %s/CACHEDIR.TAG (%s)\n",

CCache/ccache.h

@@ -92,10 +92,13 @@ void cc_log(const char *format, ...);
 void fatal(const char *msg);
 
 void copy_fd(int fd_in, int fd_out);
-int copy_file(const char *src, const char *dest);
+int safe_rename(const char* oldpath, const char* newpath);
 int move_file(const char *src, const char *dest);
 int test_if_compressed(const char *filename);
 
+int commit_to_cache(const char *src, const char *dest, int hardlink);
+int retrieve_from_cache(const char *src, const char *dest, int hardlink);
+
 int create_dir(const char *dir);
 int create_cachedirtag(const char *dir);
 void x_asprintf(char **ptr, const char *format, ...);
@@ -112,11 +115,12 @@ char *x_realpath(const char *path);
 char *gnu_getcwd(void);
 int create_empty_file(const char *fname);
 const char *get_home_directory(void);
+int x_utimes(const char *filename);
 
 void stats_update(enum stats stat);
 void stats_zero(void);
 void stats_summary(void);
-void stats_tocache(size_t size);
+void stats_tocache(size_t size, size_t numfiles);
 void stats_read(const char *stats_file, unsigned counters[STATS_END]);
 int stats_set_limits(long maxfiles, long maxsize);
 size_t value_units(const char *s);
@@ -144,6 +148,7 @@ int execute(char **argv,
 	    const char *path_stdout,
 	    const char *path_stderr);
 char *find_executable(const char *name, const char *exclude_name);
+void display_execute_args(char **argv);
 
 typedef struct {
 	char **argv;
@@ -158,6 +163,8 @@ void args_pop(ARGS *args, int n);
 void args_strip(ARGS *args, const char *prefix);
 void args_remove_first(ARGS *args);
 
+extern int ccache_verbose;
+
 #if HAVE_COMPAR_FN_T
 #define COMPAR_FN_T __compar_fn_t
 #else

CCache/execute.c

@@ -36,6 +36,11 @@ int execute(char **argv,
 	if (pid == 0) {
 		int fd;
 
+		/* TODO: needs moving after possible exit() below, but before stdout is redirected */
+		if (ccache_verbose) {
+			display_execute_args(argv);
+		}
+
 		unlink(path_stdout);
 		fd = open(path_stdout, O_WRONLY|O_CREAT|O_TRUNC|O_EXCL|O_BINARY, 0666);
 		if (fd == -1) {
@@ -127,3 +132,16 @@ char *find_executable(const char *name, const char *exclude_name)
 
 	return NULL;
 }
+
+void display_execute_args(char **argv)
+{
+	if (argv) {
+		printf("ccache executing: ");
+		while (*argv) {
+			printf("%s ", *argv);
+			++argv;
+		}
+		printf("\n");
+		fflush(stdout);
+	}
+}

CCache/stats.c

@@ -116,7 +116,7 @@ static void stats_read_fd(int fd, unsigned counters[STATS_END])
 }
 
 /* update the stats counter for this compile */
-static void stats_update_size(enum stats stat, size_t size)
+static void stats_update_size(enum stats stat, size_t size, size_t numfiles)
 {
 	int fd;
 	unsigned counters[STATS_END];
@@ -147,7 +147,7 @@ static void stats_update_size(enum stats stat, size_t size)
 
 	/* on a cache miss we up the file count and size */
 	if (stat == STATS_TOCACHE) {
-		counters[STATS_NUMFILES] += 2;
+		counters[STATS_NUMFILES] += numfiles;
 		counters[STATS_TOTALSIZE] += size;
 	}
 
@@ -173,18 +173,18 @@ static void stats_update_size(enum stats stat, size_t size)
 }
 
 /* record a cache miss */
-void stats_tocache(size_t size)
+void stats_tocache(size_t size, size_t numfiles)
 {
 	/* convert size to kilobytes */
 	size = size / 1024;
 
-	stats_update_size(STATS_TOCACHE, size);
+	stats_update_size(STATS_TOCACHE, size, numfiles);
 }
 
 /* update a normal stat */
 void stats_update(enum stats stat)
 {
-	stats_update_size(stat, 0);
+	stats_update_size(stat, 0, 0);
 }
 
 /* read in the stats from one dir and add to the counters */

CCache/util.c

@@ -44,21 +44,7 @@ void fatal(const char *msg)
 	exit(1);
 }
 
-#ifndef ENABLE_ZLIB
+int safe_rename(const char* oldpath, const char* newpath)
-/* copy all data from one file descriptor to another */
-void copy_fd(int fd_in, int fd_out)
-{
-	char buf[10240];
-	int n;
-
-	while ((n = read(fd_in, buf, sizeof(buf))) > 0) {
-		if (write(fd_out, buf, n) != n) {
-			fatal("Failed to copy fd");
-		}
-	}
-}
-
-static int safe_rename(const char* oldpath, const char* newpath)
 {
     /* safe_rename is for creating entries in the cache.
 
@@ -75,6 +61,20 @@ static int safe_rename(const char* oldpath, const char* newpath)
     }
 }
  
+#ifndef ENABLE_ZLIB
+/* copy all data from one file descriptor to another */
+void copy_fd(int fd_in, int fd_out)
+{
+	char buf[10240];
+	int n;
+
+	while ((n = read(fd_in, buf, sizeof(buf))) > 0) {
+		if (write(fd_out, buf, n) != n) {
+			fatal("Failed to copy fd");
+		}
+	}
+}
+
 /* move a file using rename */
 int move_file(const char *src, const char *dest) {
 	return safe_rename(src, dest);
@@ -83,7 +83,7 @@ int move_file(const char *src, const char *dest) {
 /* copy a file - used when hard links don't work 
    the copy is done via a temporary file and atomic rename
 */
-int copy_file(const char *src, const char *dest)
+static int copy_file(const char *src, const char *dest)
 {
 	int fd1, fd2;
 	char buf[10240];
@@ -143,6 +143,16 @@ int copy_file(const char *src, const char *dest)
 	return 0;
 }
 
+/* copy a file to the cache */
+static int copy_file_to_cache(const char *src, const char *dest) {
+	return copy_file(src, dest);
+}
+
+/* copy a file from the cache */
+static int copy_file_from_cache(const char *src, const char *dest) {
+	return copy_file(src, dest);
+}
+
 #else /* ENABLE_ZLIB */
 
 /* copy all data from one file descriptor to another
@@ -286,8 +296,13 @@ int move_file(const char *src, const char *dest) {
 	return ret;
 }
 
+/* copy a file to the cache, compressing it */
+static int copy_file_to_cache(const char *src, const char *dest) {
+	return _copy_file(src, dest, COPY_TO_CACHE);
+}
+
 /* copy a file from the cache, decompressing it */
-int copy_file(const char *src, const char *dest) {
+static int copy_file_from_cache(const char *src, const char *dest) {
 	return _copy_file(src, dest, COPY_FROM_CACHE);
 }
 #endif /* ENABLE_ZLIB */
@@ -312,6 +327,61 @@ int test_if_compressed(const char *filename) {
 	return 1;
 }
 
+/* copy file to the cache with error checking taking into account compression and hard linking if desired */
+int commit_to_cache(const char *src, const char *dest, int hardlink)
+{
+	int ret = -1;
+	unlink(dest);
+	if (hardlink) {
+		ret = link(src, dest);
+	}
+	if (ret == -1) {
+		ret = copy_file_to_cache(src, dest);
+		if (ret == -1) {
+			cc_log("failed to commit %s -> %s (%s)\n", src, dest, strerror(errno));
+			stats_update(STATS_ERROR);
+		}
+	}
+	return ret;
+}
+
+/* copy file out of the cache with error checking taking into account compression and hard linking if desired */
+int retrieve_from_cache(const char *src, const char *dest, int hardlink)
+{
+	int ret = 0;
+
+	x_utimes(src);
+
+	if (strcmp(dest, "/dev/null") == 0) {
+		ret = 0;
+	} else {
+		unlink(dest);
+		/* only make a hardlink if the cache file is uncompressed */
+		if (hardlink && test_if_compressed(src) == 0) {
+			ret = link(src, dest);
+		} else {
+			ret = copy_file_from_cache(src, dest);
+		}
+	}
+
+	/* the cached file might have been deleted by some external process */
+	if (ret == -1 && errno == ENOENT) {
+		cc_log("hashfile missing for %s\n", dest);
+		stats_update(STATS_MISSING);
+		return -1;
+	}
+
+	if (ret == -1) {
+		ret = copy_file_from_cache(src, dest);
+		if (ret == -1) {
+			cc_log("failed to retrieve %s -> %s (%s)\n", src, dest, strerror(errno));
+			stats_update(STATS_ERROR);
+			return -1;
+		}
+	}
+	return ret;
+}
+
 /* make sure a directory exists */
 int create_dir(const char *dir)
 {
@@ -410,15 +480,24 @@ void *x_malloc(size_t size)
 void *x_realloc(void *ptr, size_t size)
 {
 	void *p2;
+#if 1
+	/* Avoid invalid read in memcpy below */
+	p2 = realloc(ptr, size);
+	if (!p2) {
+		fatal("out of memory in x_realloc");
+	}
+#else
 	if (!ptr) return x_malloc(size);
 	p2 = malloc(size);
 	if (!p2) {
 		fatal("out of memory in x_realloc");
 	}
 	if (ptr) {
+		/* Note invalid read as the memcpy reads beyond the memory allocated by ptr */
 		memcpy(p2, ptr, size);
 		free(ptr);
 	}
+#endif
 	return p2;
 }
 
@@ -676,3 +755,12 @@ const char *get_home_directory(void)
 	return NULL;
 }
 
+int x_utimes(const char *filename)
+{
+#ifdef HAVE_UTIMES
+	return utimes(filename, NULL);
+#else
+	return utime(filename, NULL);
+#endif
+}
+