From: Waldemar Kozaczuk <jwkozac...@gmail.com> Committer: Nadav Har'El <n...@scylladb.com> Branch: master
aarch64: make java-base and related modules compile This patch does more than what the title suggests and ideally should be split in at least 2 parts. But it would require substantial amount of work to untangle those and more importantly re-test at this point. So firstly, this patch modifies makefiles and module.py files of java related modules and others like golang that include modules/java-base/common.gmk to make it possible to build on aarch64 host. It achieves it mostly by including modules/common.gmk which provides necessary rules for compiling on both x64 and aarch64 hosts as well as cross-compiling aarch64 on x64 host. This also removes a lot of repeated boiler plate and makes many changed makefiles more consistent with each other. Finally all build artifacts like object files are output to ./build/<release|debug>.[arch]/modules/[module] directories instead of modules/[module]. Secondly, this patch also modifies the build process to dynamically determine which jdk to use and which java tests to run. Therefore new version of modules/java/module.py detects which version of java (8 or 9 and above) is in the PATH and accordingly selects openjdk8-from-host or openjdk9_1x-from-host. In addition, we also make java-tests makefile generate list of java tests to be executed when test.py is run. This is necessary because our java "wrapper" mechanism is no longer compatible with Java 9 and above and we need to filter out java-isolated and java-non-isolated tests. New version of test.py simply reads list of java tests to be executed from modules/java-tests/test_commands, generated during build time. Signed-off-by: Waldemar Kozaczuk <jwkozac...@gmail.com> Message-Id: <20210614042047.13437-3-jwkozac...@gmail.com> --- diff --git a/modules/cloud-init/Makefile b/modules/cloud-init/Makefile --- a/modules/cloud-init/Makefile +++ b/modules/cloud-init/Makefile @@ -1,51 +1,46 @@ -SRC = $(shell readlink -f ../..) -include $(SRC)/modules/java-base/common.gmk +include ../common.gmk -autodepend = -MD -MT $@ -MP -CXXFLAGS = -g -rdynamic -Wall -std=c++11 -fPIC $(INCLUDES) $(autodepend) -src = $(SRC) -arch = x64 +module_out := $(out)/modules/cloud-init + +CXXFLAGS = -g -rdynamic -Wall -std=c++11 -fPIC $(COMMON) boost-libs := -lboost_system -lboost_filesystem -HTTPSERVER_API_DIR = ../../modules/httpserver-api -INCLUDES += -I$(HTTPSERVER_API_DIR) +HTTPSERVER_API_DIR = $(out)/modules/httpserver-api +INCLUDES += -I../httpserver-api # the build target executable: TARGET = cloud-init CPP_FILES := client.cc cloud-init.cc data-source.cc main.cc template.cc cassandra-module.cc json.cc -OBJ_FILES := $(addprefix obj/,$(CPP_FILES:.cc=.o)) +OBJ_FILES := $(addprefix $(module_out)/,$(CPP_FILES:.cc=.o)) DEPS := $(OBJ_FILES:.o=.d) STUB_HTTPSERVER_LIBS = $(HTTPSERVER_API_DIR)/httpserver-stub.so LIBS = -lpthread $(boost-libs) $(DEPEDNDS_LIBS) -lyaml-cpp -L$(HTTPSERVER_API_DIR)/ -lhttpserver-api -quiet = $(if $V, $1, @echo " $2"; $1) -very-quiet = $(if $V, $1, @$1) - - module: all -all: init $(TARGET).so tst-template +all: init $(module_out)/$(TARGET).so $(module_out)/tst-template init: - $(call very-quiet, mkdir -p obj) + $(call very-quiet, mkdir -p $(module_out)) -tst-template: template.cc tst-template.cc - $(call quiet, $(CXX) -g -Wall -std=c++11 -o $@ $^ -lboost_unit_test_framework -DBOOST_TEST_DYN_LINK, LINK $@) +$(module_out)/tst-template: template.cc tst-template.cc + $(call quiet, $(CXX) -g -Wall -std=c++11 $(LDFLAGS) -o $@ $^ -lboost_unit_test_framework -DBOOST_TEST_DYN_LINK, LINK $@) -$(TARGET): $(OBJ_FILES) - $(call quiet, $(CXX) $(CXXFLAGS) -o $(TARGET) $^ $(LIBS) $(STUB_HTTPSERVER_LIBS), LINK $@) +$(module_out)/$(TARGET): $(OBJ_FILES) + $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) $(module_out)/-o $(TARGET) $^ $(LIBS) $(STUB_HTTPSERVER_LIBS), LINK $@) -$(TARGET).so: $(OBJ_FILES) - $(call quiet, $(CXX) $(CXXFLAGS) $(STATIC_LIBS) -shared -o $(TARGET).so $^ $(LIBS), LINK $@) +$(module_out)/$(TARGET).so: $(OBJ_FILES) + $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) $(STATIC_LIBS) -shared -o $(module_out)/$(TARGET).so $^ $(LIBS), LINK $@) + echo '/usr/mgmt/cloud-init.so: ./modules/cloud-init/cloud-init.so' > usr.manifest -obj/%.o: %.cc +$(module_out)/%.o: %.cc $(call quiet, $(CXX) $(CXXFLAGS) -c -o $@ $<, CXX $@) clean: - $(call quiet, $(RM) -f $(TARGET).so tst-template, CLEAN) - $(call very-quiet, $(RM) -rf obj) + $(call quiet, $(RM) -f $(module_out)/$(TARGET).so $(module_out)/tst-template, CLEAN) + $(call very-quiet, $(RM) -rf $(module_out)) ifneq ($(MAKECMDGOALS),clean) -include $(DEPS) diff --git a/modules/cloud-init/module.py b/modules/cloud-init/module.py --- a/modules/cloud-init/module.py +++ b/modules/cloud-init/module.py @@ -6,7 +6,6 @@ _module = '${OSV_BASE}/modules/cloud-init' usr_files = FileMap() -usr_files.add(os.path.join(_module, 'cloud-init.so')).to('/usr/mgmt/cloud-init.so') usr_files.add(os.path.join(_module, 'cloud-init.yaml')).to('/usr/mgmt/cloud-init.yaml') usr_files.add(os.path.join(_module, 'cmdline')).to('/init/00-cmdline') diff --git a/modules/golang/Makefile b/modules/golang/Makefile --- a/modules/golang/Makefile +++ b/modules/golang/Makefile @@ -1,32 +1,29 @@ -SRC = $(shell readlink -f ../..) -include $(SRC)/modules/java-base/common.gmk +include ../common.gmk -autodepend = -MD -MT $@ -MP -CXXFLAGS = -g -rdynamic -Wall -std=c++11 -fPIC $(INCLUDES) $(autodepend) +module_out := $(out)/modules/golang + +CXXFLAGS = -g -rdynamic -Wall -std=c++11 -fPIC $(COMMON) # the build target executable: TARGET = go CPP_FILES := $(TARGET).cc -OBJ_FILES := $(addprefix obj/,$(CPP_FILES:.cc=.o)) +OBJ_FILES := $(addprefix $(module_out)/,$(CPP_FILES:.cc=.o)) DEPS := $(OBJ_FILES:.o=.d) -quiet = $(if $V, $1, @echo " $2"; $1) -very-quiet = $(if $V, $1, @$1) - -$(TARGET).so: $(OBJ_FILES) - $(call quiet, $(CXX) $(CXXFLAGS) -shared -o $(TARGET).so $^ $(LIBS), LINK $@) +$(module_out)/$(TARGET).so: $(OBJ_FILES) + $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared -o $(module_out)/$(TARGET).so $^ $(LIBS), LINK $@) -obj/%.o: %.cc +$(module_out)/%.o: %.cc $(call quiet, $(CXX) $(CXXFLAGS) -c -o $@ $<, CXX $@) init: @echo " MKDIRS" - $(call very-quiet, mkdir -p obj) + $(call very-quiet, mkdir -p $(module_out)) .PHONY: init -module: init $(TARGET).so - echo '/go.so: $${MODULE_DIR}/go.so' > usr.manifest +module: init $(module_out)/$(TARGET).so + echo '/go.so: ./modules/golang/go.so' > usr.manifest clean: rm -f $(TARGET)*.so usr.manifest - $(call very-quiet, $(RM) -rf obj) + $(call very-quiet, $(RM) -rf $(module_out)) diff --git a/modules/httpserver-jolokia-plugin/.gitignore b/modules/httpserver-jolokia-plugin/.gitignore --- a/modules/httpserver-jolokia-plugin/.gitignore +++ b/modules/httpserver-jolokia-plugin/.gitignore @@ -0,0 +1 @@ +usr.manifest diff --git a/modules/httpserver-jolokia-plugin/Makefile b/modules/httpserver-jolokia-plugin/Makefile --- a/modules/httpserver-jolokia-plugin/Makefile +++ b/modules/httpserver-jolokia-plugin/Makefile @@ -1,75 +1,48 @@ -SRC = $(shell readlink -f ../..) -include $(SRC)/modules/java-base/common.gmk +include ../common.gmk + +module_out := $(out)/modules/httpserver-jolokia-plugin INCLUDES += -I. -I../httpserver-api +jdkbase = $(dir $(shell readlink -f $$(which javac)))/.. +INCLUDES += -I$(jdkbase)/include -I$(jdkbase)/include/linux + # compiler flags: # -g adds debugging information to the executable file # -Wall turns on most, but not all, compiler warnings -autodepend = -MD -MT $@ -MP -CXXFLAGS = -g -Wall -std=c++11 -fPIC $(INCLUDES) -O2 $(autodepend) -src = $(shell readlink -f ../..) +CXXFLAGS = -g -Wall -std=c++11 -fPIC $(COMMON) -O2 CODE_GEN_FROM_JSON := ../httpserver-api/json2code.py -RM := /bin/rm - -ifndef ARCH - ARCH = x64 -endif - -ifndef mode - mode = release -endif - -ifndef OSV_BUILD_PATH - OSV_BUILD_PATH = $(src)/build/$(mode).$(ARCH) -endif # the build target executable: TARGET = jolokia JSON_FILES := $(wildcard api-doc/listings/*.json) JSON_CC_FILES := $(subst .json,.json.cc,$(subst api-doc/listings/,autogen/,$(JSON_FILES))) CPP_FILES := $(JSON_CC_FILES) $(wildcard *.cc) -OBJ_FILES := $(addprefix obj/,$(CPP_FILES:.cc=.o)) - -# link with -mt if present, else the base version (and hope it is multithreaded) -boost-mt := -mt -boost-lib-dir := $(dir $(shell $(CC) --print-file-name libboost_system$(boost-mt).a)) -ifeq ($(filter /%,$(boost-lib-dir)),) - boost-mt := - boost-lib-dir := $(dir $(shell $(CC) --print-file-name libboost_system$(boost-mt).a)) - ifeq ($(filter /%,$(boost-lib-dir)),) - $(error Error: libboost_system.a needs to be installed.) - endif -endif - -STATIC_LIBS = $(boost-lib-dir)/libboost_program_options$(boost-mt).a -DYN_LIBS = -lpthread -ldl -L$(libs-dir) -lyaml-cpp $(boost-libs) +OBJ_FILES := $(addprefix $(module_out)/,$(CPP_FILES:.cc=.o)) +DYN_LIBS = -lpthread -ldl -lyaml-cpp DYN_LIBS += -lssl -lcrypto LIBS = $(DYN_LIBS) $(STATIC_LIBS) -quiet = $(if $V, $1, @echo " $2"; $1) -very-quiet = $(if $V, $1, @$1) - DEPS := $(OBJ_FILES:.o=.d) module: all + echo '/usr/mgmt/plugins/jolokia.so: ./modules/httpserver-jolokia-plugin/jolokia.so' > usr.manifest -all: $(TARGET).so +all: $(module_out)/$(TARGET).so cd jolokia-agent && mvn -q package -DskipTests=true init: @echo " MKDIRS" - $(call very-quiet, mkdir -p obj) - $(call very-quiet, mkdir -p obj/json) - $(call very-quiet, mkdir -p obj/api) - $(call very-quiet, mkdir -p obj/autogen) + $(call very-quiet, mkdir -p $(module_out)/json) + $(call very-quiet, mkdir -p $(module_out)/api) + $(call very-quiet, mkdir -p $(module_out)/autogen) $(call very-quiet, mkdir -p autogen) .PHONY: init -$(TARGET).so: $(OBJ_FILES) - $(call quiet, $(CXX) $(CXXFLAGS) -shared $(STATIC_LIBS) -o $@ $^ $(DYN_LIBS), LINK $@) +$(module_out)/$(TARGET).so: $(OBJ_FILES) + $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared $(STATIC_LIBS) -o $@ $^ $(DYN_LIBS), LINK $@) ifneq ($(MAKECMDGOALS),clean) -include $(DEPS) @@ -78,17 +51,17 @@ endif autogen/%.cc: api-doc/listings/% $(CODE_GEN_FROM_JSON) | init $(call quiet,./$(CODE_GEN_FROM_JSON) -f $< -ns json, GEN $@) -obj/%.o: %.cc | init +$(module_out)/%.o: %.cc | init $(call quiet, $(CXX) $(CXXFLAGS) -c -MMD -o $@ $<, CXX $@) # jolokia.cc depends on autogen/jolokia.json.hh, which needs to be # auto-generated before jolokia.cc is compiled -obj/jolokia.o: autogen/jolokia.json.cc +$(module_out)/jolokia.o: autogen/jolokia.json.cc clean: - $(call quiet, $(RM) -f $(TARGET).so, CLEAN) + $(call quiet, $(RM) -f $(module_out)/$(TARGET).so, CLEAN) $(call very-quiet, $(RM) -f usr.manifest) - $(call very-quiet, $(RM) -rf obj) + $(call very-quiet, $(RM) -rf $(module_out)) $(call very-quiet, $(RM) -rf autogen) cd jolokia-agent && mvn -q clean -rm -f dependency-reduced-pom.xml diff --git a/modules/httpserver-jolokia-plugin/module.py b/modules/httpserver-jolokia-plugin/module.py --- a/modules/httpserver-jolokia-plugin/module.py +++ b/modules/httpserver-jolokia-plugin/module.py @@ -6,6 +6,5 @@ _module = '${OSV_BASE}/modules/httpserver-jolokia-plugin' usr_files = FileMap() -usr_files.add(os.path.join(_module, 'jolokia.so')).to('/usr/mgmt/plugins/jolokia.so') usr_files.add(os.path.join(_module, 'api-doc/listings/jolokia.json')).to('/usr/mgmt/api/listings/jolokia.json') usr_files.add(os.path.join(_module, 'jolokia-agent/target/jolokia-agent.jar')).to('/usr/mgmt/jolokia-agent.jar') diff --git a/modules/java-base/Makefile b/modules/java-base/Makefile --- a/modules/java-base/Makefile +++ b/modules/java-base/Makefile @@ -1,21 +1,22 @@ +include ../common.gmk + +module_out := $(out)/modules/java-base +export module_out + include common.gmk -ifeq ($(arch),aarch64) -java-targets := -else -java-targets := obj/jni/monitor.so obj/jvm/jni_helpers.o obj/jvm/java_api.o obj/balloon/jvm_balloon.o -endif +java-targets := $(module_out)/jni/monitor.so $(module_out)/jvm/jni_helpers.o $(module_out)/jvm/java_api.o $(module_out)/balloon/jvm_balloon.o module: all all: $(init) $(java-targets) init: @echo " MKDIRS" - $(call very-quiet, mkdir -p obj/jni) - $(call very-quiet, mkdir -p obj/jvm) - $(call very-quiet, mkdir -p obj/balloon) + $(call very-quiet, mkdir -p $(module_out)/jni) + $(call very-quiet, mkdir -p $(module_out)/jvm) + $(call very-quiet, mkdir -p $(module_out)/balloon) .PHONY: init clean: - $(call very-quiet, $(RM) -rf obj) + $(call very-quiet, $(RM) -rf $(module_out)) diff --git a/modules/java-base/common.gmk b/modules/java-base/common.gmk --- a/modules/java-base/common.gmk +++ b/modules/java-base/common.gmk @@ -1,45 +1,29 @@ jdkbase = $(dir $(shell readlink -f $$(which javac)))/.. -INCLUDES = -I$(src)/arch/$(arch) -I$(src) -I$(src)/include -I$(src)/arch/common -INCLUDES += -I$(src)/include/glibc-compat -INCLUDES += $(shell $(CXX) -E -xc++ - -v </dev/null 2>&1 | awk '/^End/ {exit} /^ .*c\+\+/ {print "-isystem" $$0}') -INCLUDES += -isystem $(src)/include/api -INCLUDES += -isystem $(src)/include/api/$(arch) -INCLUDES += -I$(src)/build/$(mode)/gen/include INCLUDES += -I$(src)/java INCLUDES += -I$(jdkbase)/include -I$(jdkbase)/include/linux -autodepend = -MD -MT $@ -MP -COMMON_FLAGS = -g -Wall -fPIC $(INCLUDES) -O2 $(autodepend) -DCONF_debug_memory=0 -D_KERNEL -CXXFLAGS = -std=c++11 $(COMMON_FLAGS) -CFLAGS = -std=gnu99 $(COMMON_FLAGS) +COMMON += -g -Wall -fPIC -O2 -DCONF_debug_memory=0 -D_KERNEL -src = $(shell readlink -f ../..) -java-base-path := $(src)/modules/java-base - -RM := /bin/rm +CXXFLAGS = -std=c++11 $(COMMON) +CFLAGS = -std=gnu99 $(COMMON) -ifndef arch - arch = x64 -endif +java-base-path := $(src)/modules/java-base -ifndef mode - mode = release -endif +javac_exe_path = $(shell realpath $$(which javac)) +javac_bin_path = $(shell dirname $(javac_exe_path)) +java_jdk_path = $(shell dirname $(javac_bin_path)) configuration-defines = conf-preempt conf-debug_memory conf-logger_debug configuration = $(foreach cf,$(configuration-defines), \ -D$(cf:conf-%=CONF_%)=$($(cf))) -quiet = $(if $V, $1, @echo " $2"; $1) -very-quiet = $(if $V, $1, @$1) - -obj/%.o: %.cc | init +$(module_out)/%.o: %.cc | init $(call quiet, $(CXX) $(CXXFLAGS) -c -MMD -o $@ $<, CXX $@) -obj/%.o: %.c | init +$(module_out)/%.o: %.c | init $(call quiet, $(CC) $(CFLAGS) -c -MMD -o $@ $<, CC $@) -%.so: %.o - $(call quiet, $(CXX) $(CXXFLAGS) -shared -o $@ $^, LINK $@) +$(module_out)/%.so: %.o | init + $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared -o $@ $^, LINK $@) diff --git a/modules/java-base/java.cc b/modules/java-base/java.cc --- a/modules/java-base/java.cc +++ b/modules/java-base/java.cc @@ -29,7 +29,11 @@ extern size_t jvm_heap_size; // sets up the class path, and runs the jar or class specified in these // parameters. +#ifdef __aarch64__ +#define JVM_PATH "/usr/lib/jvm/jre/lib/aarch64/server/libjvm.so" +#else #define JVM_PATH "/usr/lib/jvm/jre/lib/amd64/server/libjvm.so" +#endif #define JVM9_PATH "/usr/lib/jvm/java/lib/server/libjvm.so" #if defined(RUN_JAVA_NON_ISOLATED) diff --git a/modules/java-base/usr.manifest b/modules/java-base/usr.manifest --- a/modules/java-base/usr.manifest +++ b/modules/java-base/usr.manifest @@ -6,4 +6,4 @@ # [manifest] -/usr/lib/jni/monitor.so: ${MODULE_DIR}/obj/jni/monitor.so +/usr/lib/jni/monitor.so: ./modules/java-base/jni/monitor.so diff --git a/modules/java-isolated/Makefile b/modules/java-isolated/Makefile --- a/modules/java-isolated/Makefile +++ b/modules/java-isolated/Makefile @@ -1,24 +1,23 @@ -SRC = $(shell readlink -f ../..) +include ../common.gmk -include $(SRC)/modules/java-base/common.gmk +module_out := $(out)/modules/java-isolated +export module_out -ifeq ($(arch),aarch64) -java-targets := -else -java-targets := obj/java.so -endif +include ../java-base/common.gmk + +java-targets := $(module_out)/java.so jar-targets := $(java-base-path)/runjava-isolated/target/runjava-isolated.jar -obj/java.o: $(java-base-path)/java.cc | init +$(module_out)/java.o: $(java-base-path)/java.cc | init $(call quiet, $(CXX) $(CXXFLAGS) -o $@ -c $(java-base-path)/java.cc -MMD, CXX $@) -obj/java.so: obj/java.o $(java-base-path)/obj/jvm/java_api.o $(java-base-path)/obj/jvm/jni_helpers.o $(java-base-path)/obj/balloon/jvm_balloon.o - $(call quiet, $(CXX) $(CXXFLAGS) -shared -o $@ $^, LINK $@) +$(module_out)/java.so: $(module_out)/java.o $(out)/modules/java-base/jvm/java_api.o $(out)/modules/java-base/jvm/jni_helpers.o $(out)/modules/java-base/balloon/jvm_balloon.o + $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared -o $@ $^, LINK $@) init: @echo " MKDIRS" - $(call very-quiet, mkdir -p obj) + $(call very-quiet, mkdir -p $(module_out)) comma := , @@ -29,11 +28,11 @@ $(jar-targets): $(wildcard $(java-base-path)/runjava-isolated/src/main/java/io/o $(wildcard $(java-base-path)/runjava-common/src/main/java/io/osv/util/*.java) \ $(java-base-path)/pom.xml $(java-base-path)/runjava-common/pom.xml \ $(java-base-path)/runjava-isolated/pom.xml - $(call quiet, cd $(java-base-path) && mvn -q --projects :runjava-common$(comma):runjava-isolated package -DskipTests=true, MVN $@) + $(call quiet, cd $(java-base-path) && JAVA_HOME=$(java_jdk_path) mvn -q --projects :runjava-common$(comma):runjava-isolated package -DskipTests=true, MVN $@) module: $(java-targets) $(jar-targets) clean: cd $(java-base-path) && mvn -q clean -rm -f dependency-reduced-pom.xml - $(call very-quiet, $(RM) -rf obj) + $(call very-quiet, $(RM) -rf $(module_out)) diff --git a/modules/java-isolated/usr.manifest b/modules/java-isolated/usr.manifest --- a/modules/java-isolated/usr.manifest +++ b/modules/java-isolated/usr.manifest @@ -6,6 +6,6 @@ # [manifest] -/java.so: ${MODULE_DIR}/obj/java.so +/java.so: ./modules/java-isolated/java.so /java/runjava-isolated.jar: ${OSV_BASE}/modules/java-base/runjava-isolated/target/runjava-isolated.jar /.java.policy: ${MODULE_DIR}/.java.policy diff --git a/modules/java-non-isolated/Makefile b/modules/java-non-isolated/Makefile --- a/modules/java-non-isolated/Makefile +++ b/modules/java-non-isolated/Makefile @@ -1,25 +1,24 @@ -SRC = $(shell readlink -f ../..) +include ../common.gmk -include $(SRC)/modules/java-base/common.gmk +module_out := $(out)/modules/java-non-isolated +export module_out -ifeq ($(arch),aarch64) -java-targets := -else -java-targets := obj/java_non_isolated.so -endif +include ../java-base/common.gmk + +java-targets := $(module_out)/java_non_isolated.so jar-targets := $(java-base-path)/runjava-non-isolated/target/runjava-non-isolated.jar -obj/java_non_isolated.o: $(java-base-path)/java.cc | init +$(module_out)/java_non_isolated.o: $(java-base-path)/java.cc | init $(call quiet, $(CXX) $(CXXFLAGS) -DRUN_JAVA_NON_ISOLATED -o $@ -c $(java-base-path)/java.cc -MMD, CXX $@) -obj/java_non_isolated.so: obj/java_non_isolated.o $(java-base-path)/obj/jvm/java_api.o \ - $(java-base-path)/obj/jvm/jni_helpers.o $(java-base-path)/obj/balloon/jvm_balloon.o - $(call quiet, $(CXX) $(CXXFLAGS) -shared -o $@ $^, LINK $@) +$(module_out)/java_non_isolated.so: $(module_out)/java_non_isolated.o $(out)/modules/java-base/jvm/java_api.o \ + $(out)/modules/java-base/jvm/jni_helpers.o $(out)/modules/java-base/balloon/jvm_balloon.o + $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared -o $@ $^, LINK $@) init: @echo " MKDIRS" - $(call very-quiet, mkdir -p obj) + $(call very-quiet, mkdir -p $(module_out)) comma := , @@ -28,11 +27,11 @@ $(jar-targets): $(wildcard $(java-base-path)/runjava-non-isolated/src/main/java/ $(wildcard $(java-base-path)/runjava-common/src/main/java/io/osv/util/*.java) \ $(java-base-path)/pom.xml $(java-base-path)/runjava-common/pom.xml \ $(java-base-path)/runjava-non-isolated/pom.xml - $(call quiet, cd $(java-base-path) && mvn -q --projects :runjava-common$(comma):runjava-non-isolated package -DskipTests=true, MVN $@) + $(call quiet, cd $(java-base-path) && JAVA_HOME=$(java_jdk_path) mvn -q --projects :runjava-common$(comma):runjava-non-isolated package -DskipTests=true, MVN $@) module: $(java-targets) $(jar-targets) clean: cd $(java-base-path) && mvn -q clean -rm -f dependency-reduced-pom.xml - $(call very-quiet, $(RM) -rf obj) + $(call very-quiet, $(RM) -rf $(module_out)) diff --git a/modules/java-non-isolated/usr.manifest b/modules/java-non-isolated/usr.manifest --- a/modules/java-non-isolated/usr.manifest +++ b/modules/java-non-isolated/usr.manifest @@ -6,6 +6,6 @@ # [manifest] -/java.so: ${MODULE_DIR}/obj/java_non_isolated.so +/java.so: ./modules/java-non-isolated/java_non_isolated.so /java/runjava-non-isolated.jar: ${OSV_BASE}/modules/java-base/runjava-non-isolated/target/runjava-non-isolated.jar /.java.policy: ${MODULE_DIR}/.java.policy diff --git a/modules/java-tests/.gitignore b/modules/java-tests/.gitignore --- a/modules/java-tests/.gitignore +++ b/modules/java-tests/.gitignore @@ -0,0 +1 @@ +test_commands diff --git a/modules/java-tests/Makefile b/modules/java-tests/Makefile --- a/modules/java-tests/Makefile +++ b/modules/java-tests/Makefile @@ -1,29 +1,28 @@ -SRC = $(shell readlink -f ../..) -include $(SRC)/modules/java-base/common.gmk +include ../common.gmk -ifeq ($(arch),aarch64) -java-targets := -else -java-targets := obj/java_isolated.so -endif +module_out := $(out)/modules/java-tests +export module_out -isolated-jar-target := $(java-base-path)/runjava-isolated/target/runjava-isolated.jar +include ../java-base/common.gmk +javac_version := $(shell javac -version 2>&1 | grep -oP "javac \d+\.\d+") + +ifeq ($(javac_version),javac 1.8) tests-jar-target := tests/target/runjava-tests.jar +java-targets := $(module_out)/java_isolated.so + +isolated-jar-target := $(java-base-path)/runjava-isolated/target/runjava-isolated.jar + tests-isolates-target := tests-isolates/target/tests-isolates-jar-with-dependencies.jar tests-jre-extension-target := tests-jre-extension/target/tests-jre-extension.jar -obj/java_isolated.o: $(SRC)/modules/java-base/java.cc | init - $(call quiet, $(CXX) $(CXXFLAGS) -o $@ -c $(SRC)/modules/java-base/java.cc -MMD, CXX $@) - -obj/java_isolated.so: obj/java_isolated.o $(java-base-path)/obj/jvm/java_api.o $(java-base-path)/obj/jvm/jni_helpers.o $(java-base-path)/obj/balloon/jvm_balloon.o - $(call quiet, $(CXX) $(CXXFLAGS) -shared -o $@ $^, LINK $@) +$(module_out)/java_isolated.o: $(src)/modules/java-base/java.cc | init + $(call quiet, $(CXX) $(CXXFLAGS) -o $@ -c $(src)/modules/java-base/java.cc -MMD, CXX $@) -init: - @echo " MKDIRS" - $(call very-quiet, mkdir -p obj) +$(module_out)/java_isolated.so: $(module_out)/java_isolated.o $(out)/modules/java-base/jvm/java_api.o $(out)/modules/java-base/jvm/jni_helpers.o $(out)/modules/java-base/balloon/jvm_balloon.o + $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared -o $@ $^, LINK $@) comma := , @@ -34,27 +33,67 @@ $(isolated-jar-target): $(wildcard $(java-base-path)/runjava-isolated/src/main/j $(wildcard $(java-base-path)/runjava-common/src/main/java/io/osv/util/*.java) \ $(java-base-path)/pom.xml $(java-base-path)/runjava-common/pom.xml \ $(java-base-path)/runjava-isolated/pom.xml - $(call quiet, cd $(java-base-path) && mvn -q --projects :runjava-common$(comma):runjava-isolated package -DskipTests=true, MVN $@) + $(call quiet, cd $(java-base-path) && JAVA_HOME=$(java_jdk_path) mvn -q --projects :runjava-common$(comma):runjava-isolated package -DskipTests=true, MVN $@) $(tests-jre-extension-target): $(wildcard tests-jre-extension/src/main/java/tests/*.java) \ tests-jre-extension/pom.xml pom.xml - $(call quiet, cd $(SRC)/modules/java-tests && mvn -q --projects :tests-jre-extension package -DskipTests=true, MVN $@) + $(call quiet, cd $(src)/modules/java-tests && JAVA_HOME=$(java_jdk_path) mvn -q --projects :tests-jre-extension package -DskipTests=true, MVN $@) $(tests-isolates-target): $(isolated-jar-target) $(wildcard tests-isolates/src/main/java/tests/*.java) \ pom.xml tests-isolates/pom.xml - $(call quiet, cd $(SRC)/modules/java-tests && \ - mvn -q --projects :runjava-common$(comma):runjava-non-isolated$(comma):runjava-isolated$(comma):tests-isolates package -DskipTests=true, MVN $@) + $(call quiet, cd $(src)/modules/java-tests && \ + JAVA_HOME=$(java_jdk_path) mvn -q --projects :runjava-common$(comma):runjava-non-isolated$(comma):runjava-isolated$(comma):tests-isolates package -DskipTests=true, MVN $@) $(tests-jar-target): $(isolated-jar-target) $(tests-isolates-target) $(tests-jre-extension-target) \ $(wildcard tests/src/main/java/io/osv/*.java) \ $(wildcard tests/src/main/java/tests/*.java) \ pom.xml tests/pom.xml - $(call quiet, cd $(SRC)/modules/java-tests && \ - mvn -q --projects :runjava-common$(comma):runjava-non-isolated$(comma):runjava-isolated$(comma):tests-isolates$(comma):runjava-tests$(comma):tests-jre-extension package -DskipTests=true, MVN $@) + $(call quiet, cd $(src)/modules/java-tests && \ + JAVA_HOME=$(java_jdk_path) mvn -q --projects :runjava-common$(comma):runjava-non-isolated$(comma):runjava-isolated$(comma):tests-isolates$(comma):runjava-tests$(comma):tests-jre-extension package -DskipTests=true, MVN $@) + +init: + @echo " MKDIRS" + $(call very-quiet, mkdir -p $(module_out)) + +module: $(java-targets) $(tests-jar-target) test_commands +else +tests-jar-target := tests-for-java9_1x/target/runjava-9-1x-tests.jar + +$(tests-jar-target): $(wildcard $(java-base-path)/tests-for-java9_1x/src/main/java/io/osv//*.java) + $(call quiet, cd $(src)/modules/java-tests/tests-for-java9_1x && \ + JAVA_HOME=$(java_jdk_path) mvn -q package -DskipTests=true, MVN $@) -module: $(java-targets) $(tests-jar-target) +module: $(tests-jar-target) test_commands +endif + +ifeq ($(arch),aarch64) +java_arch_options := -Xint +endif + +java_isolated_cmd := 'java_isolated: /java_isolated.so $(java_arch_options) -cp /tests/java/tests.jar:/tests/java/isolates.jar \ + -Disolates.jar=/tests/java/isolates.jar org.junit.runner.JUnitCore io.osv.AllTestsThatTestIsolatedApp' +java_non_isolated_cmd := 'java_non_isolated: /java.so $(java_arch_options) -cp /tests/java/tests.jar:/tests/java/isolates.jar \ + -Disolates.jar=/tests/java/isolates.jar org.junit.runner.JUnitCore io.osv.AllTestsThatTestNonIsolatedApp' +java_no_wrapper_cmd := 'java_no_wrapper: /usr/bin/java $(java_arch_options) -cp /tests/java/tests.jar org.junit.runner.JUnitCore io.osv.BasicTests !' +java_perms_cmd := 'java-perms: /usr/bin/java $(java_arch_options) -cp /tests/java/tests.jar io.osv.TestDomainPermissions !' + +.PHONY: test_commands + +ifeq ($(arch),$(host_arch)) +test_commands: + $(call very-quiet, rm -f test_commands) + $(call very-quiet, if [ "$(javac_version)" = "javac 1.8" ]; then \ + echo $(java_isolated_cmd) >> test_commands && \ + echo $(java_non_isolated_cmd) >> test_commands; fi ) + $(call very-quiet, echo $(java_no_wrapper_cmd) >> test_commands) + $(call very-quiet, echo $(java_perms_cmd) >> test_commands) +else +test_commands: + $(call very-quiet, rm -f test_commands) +endif clean: - cd $(SRC)/modules/java-tests && mvn -q clean - -rm -f dependency-reduced-pom.xml - $(call very-quiet, $(RM) -rf obj) + cd $(src)/modules/java-tests && mvn -q clean + cd $(src)/modules/java-tests/tests-for-java9_1x && mvn -q clean + -rm -f dependency-reduced-pom.xml test_commands + $(call very-quiet, $(RM) -rf $(module_out)) diff --git a/modules/java-tests/module.py b/modules/java-tests/module.py --- a/modules/java-tests/module.py +++ b/modules/java-tests/module.py @@ -1,16 +1,27 @@ from osv.modules import api from osv.modules.filemap import FileMap +import subprocess api.require('java') -_jar = '/tests/java/tests.jar' -_isolates_jar = '/tests/java/isolates.jar' +javac_with_version = subprocess.check_output(['javac', '-version'], stderr=subprocess.STDOUT).decode('utf-8') usr_files = FileMap() -usr_files.add('${OSV_BASE}/modules/java-tests/tests/target/runjava-tests.jar').to(_jar) -usr_files.add('${OSV_BASE}/modules/java-tests/tests-isolates/target/tests-isolates-jar-with-dependencies.jar').to(_isolates_jar) +_jar = '/tests/java/tests.jar' + +if javac_with_version.startswith('javac 1.8'): + _isolates_jar = '/tests/java/isolates.jar' + + usr_files.add('${OSV_BASE}/modules/java-tests/tests/target/runjava-tests.jar').to(_jar) + usr_files.add('${OSV_BASE}/modules/java-tests/tests-isolates/target/tests-isolates-jar-with-dependencies.jar').to(_isolates_jar) + + usr_files.add('${OSV_BASE}/modules/java-tests/tests/target/classes/tests/ClassPutInRoot.class').to('/tests/ClassPutInRoot.class') -usr_files.add('${OSV_BASE}/modules/java-tests/tests/target/classes/tests/ClassPutInRoot.class').to('/tests/ClassPutInRoot.class') + usr_files.add('${OSV_BASE}/modules/java-tests/tests-jre-extension/target/tests-jre-extension.jar') \ + .to('/usr/lib/jvm/java/jre/lib/ext/tests-jre-extension.jar') -usr_files.add('${OSV_BASE}/modules/java-tests/tests-jre-extension/target/tests-jre-extension.jar') \ - .to('/usr/lib/jvm/java/jre/lib/ext/tests-jre-extension.jar') + usr_files.add('${OSV_BASE}/build/last/modules/java-tests/java_isolated.so').to('/java_isolated.so') + usr_files.add('${OSV_BASE}/modules/java-base/runjava-isolated/target/runjava-isolated.jar').to('/java/runjava-isolated.jar') + usr_files.add('${OSV_BASE}/modules/java-tests/.java.policy').to('/.java.policy') +else: + usr_files.add('${OSV_BASE}/modules/java-tests/tests-for-java9_1x/target/runjava-9-1x-tests.jar').to(_jar) diff --git a/modules/java-tests/tests-for-java9_1x/pom.xml b/modules/java-tests/tests-for-java9_1x/pom.xml --- a/modules/java-tests/tests-for-java9_1x/pom.xml +++ b/modules/java-tests/tests-for-java9_1x/pom.xml @@ -0,0 +1,64 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <groupId>io.osv</groupId> + <artifactId>runjava-9-1x-tests</artifactId> + <version>1.0-SNAPSHOT</version> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.13.1</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.3</version> + <executions> + <execution> + <goals> + <goal>properties</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <finalName>runjava-9-1x-tests</finalName> + <appendAssemblyId>false</appendAssemblyId> + </configuration> + <executions> + <execution> + <id>make-jar-with-dependencies</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.1</version> + <configuration> + <source>1.7</source> + <target>1.7</target> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/BasicTest.java b/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/BasicTest.java --- a/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/BasicTest.java +++ b/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/BasicTest.java @@ -0,0 +1 @@ +../../../../../../tests/src/main/java/io/osv/BasicTest.java \ No newline at end of file diff --git a/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/BasicTests.java b/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/BasicTests.java --- a/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/BasicTests.java +++ b/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/BasicTests.java @@ -0,0 +1 @@ +../../../../../../tests/src/main/java/io/osv/BasicTests.java \ No newline at end of file diff --git a/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/TemporarySecurityManager.java b/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/TemporarySecurityManager.java --- a/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/TemporarySecurityManager.java +++ b/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/TemporarySecurityManager.java @@ -0,0 +1 @@ +../../../../../../tests/src/main/java/io/osv/TemporarySecurityManager.java \ No newline at end of file diff --git a/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/TestDomainPermissions.java b/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/TestDomainPermissions.java --- a/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/TestDomainPermissions.java +++ b/modules/java-tests/tests-for-java9_1x/src/main/java/io/osv/TestDomainPermissions.java @@ -0,0 +1 @@ +../../../../../../tests/src/main/java/io/osv/TestDomainPermissions.java \ No newline at end of file diff --git a/modules/java-tests/usr.manifest b/modules/java-tests/usr.manifest --- a/modules/java-tests/usr.manifest +++ b/modules/java-tests/usr.manifest @@ -1,11 +0,0 @@ -# -# Copyright (C) 2013-2014 Cloudius Systems, Ltd. -# -# This work is open source software, licensed under the terms of the -# BSD license as described in the LICENSE file in the top-level directory. -# - -[manifest] -/java_isolated.so: ${OSV_BASE}/modules/java-tests/obj/java_isolated.so -/java/runjava-isolated.jar: ${OSV_BASE}/modules/java-base/runjava-isolated/target/runjava-isolated.jar -/.java.policy: ${OSV_BASE}/modules/java-tests/.java.policy diff --git a/modules/java/module.py b/modules/java/module.py --- a/modules/java/module.py +++ b/modules/java/module.py @@ -1,4 +1,10 @@ from osv.modules import api +import subprocess -api.require('java-non-isolated') -api.require('openjdk8-from-host') +javac_with_version = subprocess.check_output(['javac', '-version'], stderr=subprocess.STDOUT).decode('utf-8') + +if javac_with_version.startswith('javac 1.8'): + api.require('java-non-isolated') + api.require('openjdk8-from-host') +else: + api.require('openjdk9_1x-from-host') diff --git a/modules/josvsym/Makefile b/modules/josvsym/Makefile --- a/modules/josvsym/Makefile +++ b/modules/josvsym/Makefile @@ -1,60 +1,40 @@ -SRC = $(shell readlink -f ../..) -include $(SRC)/modules/java-base/common.gmk -INCLUDES += -I. +include ../common.gmk + +module_out := $(out)/modules/josvsym + +jdkbase = $(dir $(shell readlink -f $$(which javac)))/.. +INCLUDES += -I$(jdkbase)/include -I$(jdkbase)/include/linux # compiler flags: # -g adds debugging information to the executable file # -Wall turns on most, but not all, compiler warnings -autodepend = -MD -MT $@ -MP -CXXFLAGS = -g -Wall -std=c++11 -fPIC $(INCLUDES) -O2 $(autodepend) - -src = $(shell readlink -f ../..) - -ifndef ARCH - ARCH = x64 -endif - -ifndef mode - mode = release -endif - -ifndef OSV_BUILD_PATH - OSV_BUILD_PATH = $(src)/build/$(mode).$(ARCH) -endif +CXXFLAGS = -g -Wall -std=c++11 -fPIC $(COMMON) -O2 MODULE = josvsym -OUTDIR := $(OSV_BUILD_PATH)/modules/$(MODULE) CPP_FILES := josvsym.cc -OBJ_FILES := $(addprefix $(OUTDIR)/,$(CPP_FILES:.cc=.o)) +OBJ_FILES := $(addprefix $(module_out)/,$(CPP_FILES:.cc=.o)) DEPS := $(OBJ_FILES:.o=.d) -TARGET := $(OUTDIR)/lib$(MODULE).so -LIBS = - -quiet = $(if $V, $1, @echo " $2"; $1) -very-quiet = $(if $V, $1, @$1) +TARGET := $(module_out)/lib$(MODULE).so module : all -all : $(OUTDIR) $(TARGET) +all : $(TARGET) ifneq ($(MAKECMDGOALS),clean) -include $(DEPS) endif -$(OUTDIR) : - $(call very-quiet, mkdir -p $(OUTDIR)) - -.PHONY : all module +init : + @echo " MKDIRS" + $(call very-quiet, mkdir -p $(module_out)) $(TARGET) : $(OBJ_FILES) $(STUB_FILES) - $(call quiet, $(CXX) $(CXXFLAGS) -shared -o $@ $^ $(LIBS), LINK $(@F)) + $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared -o $@ $^ $(LIBS), LINK $(@F)) -$(OUTDIR)/%.o : %.cc +$(module_out)/%.o : %.cc | init $(call quiet, $(CXX) $(CXXFLAGS) -c -MMD -o $@ $<, CXX $@) clean : - $(call quiet, $(RM) -f $(TARGET), CLEAN) - $(call very-quiet, $(RM) -rf $(OUTDIR/*.o)) - $(call very-quiet, $(RM) -rf $(OUTDIR/*.d)) + $(call quiet, $(RM) -rf $(module_out), CLEAN) diff --git a/modules/tests/module.py b/modules/tests/module.py --- a/modules/tests/module.py +++ b/modules/tests/module.py @@ -1,6 +1,5 @@ from osv.modules import api import os -if os.getenv('ARCH') == 'x64': - api.require('java-tests') +api.require('java-tests') api.require('dl_tests') diff --git a/scripts/test.py b/scripts/test.py --- a/scripts/test.py +++ b/scripts/test.py @@ -35,11 +35,6 @@ #The remaining ones are disabled below until we fix various #issues that prevent those tests from passing. aarch64_disabled_list= [ - #All java tests require JVM running on aarch64 which in turn at least requires TLS support - "java_isolated", - "java_non_isolated", - "java_no_wrapper", - "java-perms", #Following tests crash with message 'Assertion failed: type == ARCH_JUMP_SLOT (core/elf.cc: relocate_pltgot: 789)' "tst-sigaltstack.so", #Remaining tests below fail for various different reasons @@ -48,16 +43,6 @@ "tst-mmap.so", # Infinite page fault ] -add_tests([ - SingleCommandTest('java_isolated', '/java_isolated.so -cp /tests/java/tests.jar:/tests/java/isolates.jar \ - -Disolates.jar=/tests/java/isolates.jar org.junit.runner.JUnitCore io.osv.AllTestsThatTestIsolatedApp'), - SingleCommandTest('java_non_isolated', '/java.so -cp /tests/java/tests.jar:/tests/java/isolates.jar \ - -Disolates.jar=/tests/java/isolates.jar org.junit.runner.JUnitCore io.osv.AllTestsThatTestNonIsolatedApp'), - SingleCommandTest('java_no_wrapper', '/usr/bin/java -cp /tests/java/tests.jar \ - org.junit.runner.JUnitCore io.osv.BasicTests !'), - SingleCommandTest('java-perms', '/java_isolated.so -cp /tests/java/tests.jar io.osv.TestDomainPermissions'), -]) - class TestRunnerTest(SingleCommandTest): def __init__(self, name): super(TestRunnerTest, self).__init__(name, '/tests/%s' % name) @@ -86,6 +71,16 @@ def collect_tests(): test_files.append(guestpath); add_tests((TestRunnerTest(os.path.basename(x)) for x in test_files)) +def collect_java_tests(): + with open('modules/java-tests/test_commands', 'r') as f: + for line in f: + line = line.rstrip(); + if is_comment.match(line): continue; + components = line.split(": ", 2); + test_name = components[0].strip(); + test_command = components[1].strip() + add_tests([SingleCommandTest(test_name, test_command)]) + def run_test(test): sys.stdout.write(" TEST %-35s" % test.name) sys.stdout.flush() @@ -154,6 +149,7 @@ def run_tests(): print(("OK (%d %s run, %.3f s)" % (len(tests_to_run), pluralize("test", len(tests_to_run)), duration))) def main(): + collect_java_tests() collect_tests() while True: run_tests() -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/000000000000eed25605c4b75c86%40google.com.