Rework compress/uncompress targets
- now users can define their own COMPRESS/UNCOMPRESS commands
- by default any compressed files that are still larger than 100 MB will
be split automatically to avoid the need for LFS storage
diff --git a/Makefile b/Makefile
index 52964a6..a060310 100644
--- a/Makefile
+++ b/Makefile
@@ -16,13 +16,31 @@
# cannot commit files larger than 100 MB to GitHub
FILE_SIZE_LIMIT_MB = 100
+
+# Commands to be used to compress/uncompress files
+# they must operate **in place** (otherwise, modify the target to delete the
+# intermediate file/archive)
+COMPRESS ?= gzip -n --best
+UNCOMPRESS ?= gzip -d
+ARCHIVE_EXT ?= gz
+
+# The following variables are to build static pattern rules
+
+# Needed to rebuild archives that were previously split
+SPLIT_FILES := $(shell find . -type f -name "*.$(ARCHIVE_EXT).00.split")
+SPLIT_FILES_SOURCES := $(basename $(basename $(basename $(SPLIT_FILES))))
+
+# Needed to uncompress the existing archives
+ARCHIVES := $(shell find . -type f -name "*.$(ARCHIVE_EXT)")
+ARCHIVE_SOURCES := $(basename $(ARCHIVES))
+
+# Needed to compress and split files/archives that are too large
LARGE_FILES := $(shell find ./gds -type f -name "*.gds")
LARGE_FILES += $(shell find . -type f -size +$(FILE_SIZE_LIMIT_MB)M -not -path "./.git/*" -not -path "./gds/*" -not -path "./openlane/*")
-
-LARGE_FILES_GZ := $(addsuffix .gz, $(LARGE_FILES))
-
-ARCHIVES := $(shell find . -type f -name "*.gz")
-ARCHIVE_SOURCES := $(basename $(ARCHIVES))
+LARGE_FILES_GZ := $(addsuffix .$(ARCHIVE_EXT), $(LARGE_FILES))
+LARGE_FILES_GZ_SPLIT := $(addsuffix .$(ARCHIVE_EXT).00.split, $(LARGE_FILES))
+# consider splitting existing archives
+LARGE_FILES_GZ_SPLIT += $(addsuffix .00.split, $(ARCHIVES))
# PDK setup configs
THREADS ?= $(shell nproc)
@@ -61,25 +79,43 @@
-$(LARGE_FILES_GZ): %.gz: %
- @if ! [ $(suffix $<) == ".gz" ]; then\
- gzip -n --best $< > /dev/null &&\
+#####
+$(LARGE_FILES_GZ): %.$(ARCHIVE_EXT): %
+ @if ! [ $(suffix $<) == ".$(ARCHIVE_EXT)" ]; then\
+ $(COMPRESS) $< > /dev/null &&\
echo "$< -> $@";\
fi
+$(LARGE_FILES_GZ_SPLIT): %.$(ARCHIVE_EXT).00.split: %.$(ARCHIVE_EXT)
+ @if [ -n "$$(find "$<" -prune -size +$(FILE_SIZE_LIMIT_MB)M)" ]; then\
+ split $< -b $(FILE_SIZE_LIMIT_MB)M $<. -d --additional-suffix=.split &&\
+ rm $< &&\
+ echo -n "$< -> $$(ls $<.*.split)" | tr '\n' ' ' && echo "";\
+ fi
+
# This target compresses all files larger than $(FILE_SIZE_LIMIT_MB) MB
.PHONY: compress
-compress: $(LARGE_FILES_GZ)
+compress: $(LARGE_FILES_GZ) $(LARGE_FILES_GZ_SPLIT)
@echo "Files larger than $(FILE_SIZE_LIMIT_MB) MBytes are compressed!"
-$(ARCHIVE_SOURCES): %: %.gz
- @gzip -d $< &&\
- echo "$< -> $@";\
+#####
+$(ARCHIVE_SOURCES): %: %.$(ARCHIVE_EXT)
+ @$(UNCOMPRESS) $<
+ @echo "$< -> $@"
+
+.SECONDEXPANSION:
+$(SPLIT_FILES_SOURCES): %: $$(sort $$(wildcard %.$(ARCHIVE_EXT).*.split))
+ @cat $? > $@.$(ARCHIVE_EXT)
+ @rm $?
+ @echo "$? -> $@.$(ARCHIVE_EXT)"
+ @$(UNCOMPRESS) $@.$(ARCHIVE_EXT)
+ @echo "$@.$(ARCHIVE_EXT) -> $@"
+
.PHONY: uncompress
-uncompress: $(ARCHIVE_SOURCES)
+uncompress: $(SPLIT_FILES_SOURCES) $(ARCHIVE_SOURCES)
@echo "All files are uncompressed!"
@@ -189,7 +225,7 @@
help:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$'
-
+
###########################################################################
.PHONY: pdk
pdk: skywater-pdk skywater-library skywater-timing open_pdks build-pdk
