[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville abandoned this revision.
winksaville added a comment.

Abandoning, @beanz has proposed a better solution, D61909 



Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville added a comment.

Sorry, the two previous comments were meant for D61909 
 and I've moved them.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville added a comment.

Questions:

- Should we only build `libclang_shared.so` if `LLVM_BUILD_LLVM_DYLIB` is ON?
- Should we use link clang-9 to libclang_shared.so when `LLVM_LINK_LLVM_DYLIB` 
is ON?
- Or maybe we should define `BUILD_CLANG_DYLIB` and `LINK_CLANG_DYLIB` or ... ?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville added a comment.

I did some quick testing.

I used cmake and ninja to build `llvm` and enabled `clang;lld;compiler-rt` 
subprojects:

  $ cd build-beanz-clang-shlib-2-add-debug-BUILD-LINK_DYNLIB-ON
  $ cmake ../llvm -G Ninja '-DLLVM_ENABLE_PROJECTS=clang;lld;compiler-rt' 
-DCMAKE_INSTALL_PREFIX=/home/wink/local-beanz-clang-shlib-2-add-debug-BUILD-LINK_DYNLIB-ON
 -DCMAKE_BUILD_TYPE=Release -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_LINK_LLVM_DYLIB=ON
  $ ninja

I then used `bin/clang++` to create `bin/clang-shared` and verified it ran and 
looked at the sizes; `bin/clang-shared` is 187K while `bin/clang-9` is 45M

  $ bin/clang++  -fPIC -fvisibility-inlines-hidden -Werror=date-time -std=c++11 
-Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual 
-Wno-missing-field-initializers -pedantic -Wno-long-long -Wimplicit-fallthrough 
-Wno-maybe-uninitialized -Wno-class-memaccess -Wno-noexcept-type 
-Wdelete-non-virtual-dtor -Wno-comment -fdiagnostics-color -ffunction-sections 
-fdata-sections -fno-common -Woverloaded-virtual -fno-strict-aliasing -O3 
-DNDEBUG  -Wl,-allow-shlib-undefined   -Wl,--export-dynamic  
-Wl,-rpath-link,/home/wink/prgs/llvm/llvm-project/build-beanz-clang-shlib-2-add-debug-BUILD-LINK_DYNLIB-ON/./lib
  -Wl,-O3 tools/clang/tools/driver/CMakeFiles/clang.dir/driver.cpp.o 
tools/clang/tools/driver/CMakeFiles/clang.dir/cc1_main.cpp.o 
tools/clang/tools/driver/CMakeFiles/clang.dir/cc1as_main.cpp.o 
tools/clang/tools/driver/CMakeFiles/clang.dir/cc1gen_reproducer_main.cpp.o  -o 
bin/clang-shared  -Wl,-rpath,"\$ORIGIN/../lib" -lpthread lib/libclang_shared.so 
lib/libLLVM-9svn.so 
  $ ./bin/clang-shared --version
  clang version 9.0.0 (g...@github.com:winksaville/llvm-project 
2e41f82e327c9aace5b1367995357a787be77105)
  Target: x86_64-unknown-linux-gnu
  Thread model: posix
  InstalledDir: 
/home/wink/prgs/llvm/llvm-project/build-beanz-clang-shlib-2-add-debug-BUILD-LINK_DYNLIB-ON/./bin
  $ ls -hal bin/clang-9 bin/clang-shared
  -rwxr-xr-x 1 wink users  45M May 14 17:57 bin/clang-9
  -rwxr-xr-x 1 wink users 187K May 14 22:15 bin/clang-shared

And I'd previously built `clang-9` with LLVM_BUILD_LLVM_DYLIB=OFF and it is 
114M:

  $ ls -hal ../build-beanz-clang-shlib-2-add-debug-DYNLIB-OFF/bin/clang-9
  -rwxr-xr-x 1 wink users 114M May 14 11:31 
../build-beanz-clang-shlib-2-add-debug-DYNLIB-OFF/bin/clang-9

I then compilef hi.c with `clang-shared` and ran the result:

  $ cat ../hi.c
  #include 
  
  int main(void) {
printf("hi\n");
return 0;
  }
  $ ./bin/clang-shared ../hi.c -o hi
  $ ./hi
  hi

LGTM


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Chris Bieneman via Phabricator via cfe-commits]

beanz added a comment.

@winksaville that was my bad. I left off the new files in the commit because I 
forgot `git-diff` doesn't grab untracked files...

I've uploaded the patch as a D61909 .


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Fangrui Song via Phabricator via cfe-commits]

MaskRay added a comment.

@winksaville I asked why on Arch Linux, extra/clang is built with 
`-DBUILD_SHARED_LIBS=ON` while extra/llvm extra/lldb are built with 
`-DBUILD_SHARED_LIBS=ON` https://bugs.archlinux.org/task/60512 . I think 
`extra/clang` is just not suitable for development. If you really want to build 
static libraries than convert them to shared libraries, there is a hack 
...


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Tom Stellard via Phabricator via cfe-commits]

tstellar added a comment.

> ! In D61804#149 , @beanz wrote:
>  I apologize that I missed your thread on the dev list, because that would 
> have been a much better place to have this conversation. Having gone back and 
> read it now, it sounds to me like what you really want is a clang equivalent 
> of libLLVM. That is a reasonable and viable thing to want. This patch 
> (https://reviews.llvm.org/P8147) is a first stab. I'm happy to prepare that 
> for landing in Clang if that meets your needs, and that is a viable way 
> forward.

I think this patch is missing the new clang-shlib directory.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville added a comment.

@beanz I found what I think is the reason libclang_shared.so isn't being 
created.

I added a bunch of debug output using `message` to add_llvm_subdirectory:

  1041  function(add_llvm_subdirectory project type name)
  1042message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}:+")
  1043set(add_llvm_external_dir "${ARGN}")
  1044if("${add_llvm_external_dir}" STREQUAL "")
  1045  message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 
1")
  1046  set(add_llvm_external_dir ${name})
  1047endif()
  1048canonicalize_tool_name(${name} nameUPPER)
  1049set(canonical_full_name ${project}_${type}_${nameUPPER})
  1050get_property(already_processed GLOBAL PROPERTY 
${canonical_full_name}_PROCESSED)
  1051if(already_processed)
  1052  message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}:- 
already_processed")
  1053  return()
  1054endif()
  1055set_property(GLOBAL PROPERTY ${canonical_full_name}_PROCESSED YES)
  1056message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 1.1 
canonical_full_name=${canonical_full_name} nameUPPER=${nameUPPER}")
  1057  
  1058if(EXISTS 
${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir}/CMakeLists.txt)
  1059  message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 
2")
  1060  # Treat it as in-tree subproject.
  1061  option(${canonical_full_name}_BUILD
  1062 "Whether to build ${name} as part of ${project}" On)
  1063  mark_as_advanced(${project}_${type}_${name}_BUILD)
  1064  if(${canonical_full_name}_BUILD)
  1065
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir} 
${add_llvm_external_dir})
  1066  endif()
  1067else()
  1068  set(LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR
  1069"${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR}"
  1070CACHE PATH "Path to ${name} source directory")
  1071  message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 3 
LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR=\"${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR}\"")
  1072  set(${canonical_full_name}_BUILD_DEFAULT ON)
  1073  if(NOT LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR OR NOT EXISTS 
${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR})
  1074set(${canonical_full_name}_BUILD_DEFAULT OFF)
  1075message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 
3.1 
${canonical_full_name}_BUILD_DEFAULT=${${canonical_full_name}_BUILD_DEFAULT}")
  1076  endif()
  1077  if("${LLVM_EXTERNAL_${nameUPPER}_BUILD}" STREQUAL "OFF")
  1078set(${canonical_full_name}_BUILD_DEFAULT OFF)
  1079message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 
3.2 ${LLVM_EXTERNAL_${nameUPPER}_BUILD} 
${canonical_full_name}_BUILD_DEFAULT=${${canonical_full_name}_BUILD_DEFAULT}")
  1080  endif()
  1081  option(${canonical_full_name}_BUILD
  1082"Whether to build ${name} as part of LLVM"
  1083${${canonical_full_name}_BUILD_DEFAULT})
  1084  message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 
3.3 ${canonical_full_name}_BUILD=${${canonical_full_name}_BUILD}")
  1085  if (${canonical_full_name}_BUILD)
  1086message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 
3.4")
  1087if(EXISTS ${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR})
  1088  message(STATUS "add_llvm_subdirectory ${project} ${type} 
${name}: 3.4.1")
  1089  add_subdirectory(${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR} 
${add_llvm_external_dir})
  1090elseif(NOT "${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR}" STREQUAL "")
  1091  message(STATUS "add_llvm_subdirectory ${project} ${type} 
${name}: 3.4.2")
  1092  message(WARNING "Nonexistent directory for ${name}: 
${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR}")
  1093endif()
  1094message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 
3.4.99")
  1095  endif()
  1096  message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}: 
3.99")
  1097endif()
  1098message(STATUS "add_llvm_subdirectory ${project} ${type} ${name}:-")
  1099  endfunction()

The typical output when adding a subdirectory is:

  -- add_llvm_subdirectory CLANG TOOL clang-refactor:+
  -- add_llvm_subdirectory CLANG TOOL clang-refactor: 1
  -- add_llvm_subdirectory CLANG TOOL clang-refactor: 1.1 
canonical_full_name=CLANG_TOOL_CLANG_REFACTOR nameUPPER=CLANG_REFACTOR
  -- add_llvm_subdirectory CLANG TOOL clang-refactor: 2
  -- add_llvm_subdirectory CLANG TOOL clang-refactor:-

Where at line 1058 the clause `if(EXISTS 
${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir}/CMakeLists.txt)`
is TRUE and we go through path `2`; lines 1059 - 1066.

The only time we don't go though path 2 is clang-shlib which goes though path 
3,  lines 1068 - 1096.
In that case we endup with CLANG_TOOL_CLANG_SHLIB_BUILD_DEFAULT=OFF
and CLANG_TOOL_CLANG_SHLIB_BUILD=OFF so 

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville added a comment.

In D61804#1500445 , @winksaville wrote:

> In D61804#1500409 , @beanz wrote:
>
> > My change should not have decreased build time from trunk, nor should it 
> > have reduced the number of build steps. That patch should generate 
> > lib/libClang_shared.so, which will export the C++ API. libClang is 
> > unchanged since changing it would break existing clients of the library.
>
>
> OK, I'll look into what I've done wrong. If you see anything or have any 
> guesses please let me know.


I found my mistake that caused the speed up, I'd aborted a previous build so 
ninja started where it left off, just as should :)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville added a comment.

In D61804#1500409 , @beanz wrote:

> My change should not have decreased build time from trunk, nor should it have 
> reduced the number of build steps. That patch should generate 
> lib/libClang_shared.so, which will export the C++ API. libClang is unchanged 
> since changing it would break existing clients of the library.


OK, I'll look into what I've done wrong. If you see anything or have any 
guesses please let me know.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Chris Bieneman via Phabricator via cfe-commits]

beanz added a comment.

My change should not have decreased build time from trunk, nor should it have 
reduced the number of build steps. That patch should generate 
lib/libClang_shared.so, which will export the C++ API. libClang is unchanged 
since changing it would break existing clients of the library.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville added a comment.

> Arch Linux is in an unsupported state, and your patch isn't the way forward.

OK, I'll filed a bug, https://bugs.archlinux.org/task/62624

> Let's rewind.
> 
> Why do "some people like shared libraries"? There are usually two reasons 
> people cite for linking shared libraries. One is reduced binary distribution 
> size because you're reducing duplication of code. The other, is the ability 
> to update libraries without updating tools.
> 
> The later, is not an issue here. LLVM and Clang's C++ libraries don't provide 
> any API or ABI stability guarantees across releases, so there is no way 
> you're going to reasonably update LLVM or Clang libraries without rebuilding 
> the applications using them.
> 
> The former can be valuable, but it comes at a cost. Dynamic resolution of 
> symbols slows down process launch time, and dynamic resolution of C++ 
> link-once ODRs are exceptionally slow. If you are building a compiler, 
> chances are you don't want to slow down your process launch that much. But 
> let's pretend you don't care. Your solution still isn't the right way to do 
> this.
> 
> Creating component shared libraries results in duplicating all the C++ 
> link-once ODRs across each shared module. That will not only slow down your 
> process launch, but bloat the size of your binary distribution.

Learn something new every day, thanks.

> 
> 
> In D61804#1499276 , @winksaville 
> wrote:
> 
>> It looks to me that llvm, libunwind and libcxx support building both, so the 
>> goal isn't unprecedented,
> 
> 
> This is very much an apple and oranges comparison. For one, LLVM does not 
> support generating component dylibs and shared libs. It supports generating 
> libLLVM, a single dylib containing all of the LLVM components rolled 
> together. libunwind, libcxx, libcxxabi, are runtime libraries that are 
> designed to allow static linkage, and they don't link LLVM.
> 
> I apologize that I missed your thread on the dev list, because that would 
> have been a much better place to have this conversation. Having gone back and 
> read it now, it sounds to me like what you really want is a clang equivalent 
> of libLLVM. That is a reasonable and viable thing to want. This patch 
> (https://reviews.llvm.org/P8147) is a first stab. I'm happy to prepare that 
> for landing in Clang if that meets your needs, and that is a viable way 
> forward.

Yes, as @tstellar also said, creating a single clang ".so" that exported the 
same set of symbols as the static libraries (libclang*.a) would be nice and 
probably what Arch Linux would prefer, especially if it was "officially" 
supported and was generated "simultaneously" with the static libraries.

I applied your patch to master:

  $ git log -1
  commit e5a5b7db3ac71a37af790a416486f653dbff47fb (HEAD -> master, 
origin/master, origin/HEAD)
  Author: anatolik 
  Date:   Sun May 12 19:16:54 2019 +
  
  archrelease: copy trunk to testing-x86_64
  
  git-svn-id: file:///srv/repos/svn-packages/svn@353145 
eb2447ed-0c53-47e4-bac8-5bc4a241df78

And built llvm, clang, lld and compiler-rt using:

  $ cmake ../llvm -G Ninja '-DLLVM_ENABLE_PROJECTS=clang;lld;compiler-rt' 
-DCMAKE_INSTALL_PREFIX=/home/wink/local-clang-shlib-master 
-DCMAKE_BUILD_TYPE=Release -DLLVM_BUILD_LLVM_DYLIB=ON
  $ ninja -j11 -v

I see libclang.so*:

  $ ls build-clang-shlib-master/lib/libclang.so* -al
  lrwxrwxrwx 1 wink users   16 May 13 10:49 
build-clang-shlib-master/lib/libclang.so -> libclang.so.9svn
  -rwxr-xr-x 1 wink users 81573360 May 13 10:49 
build-clang-shlib-master/lib/libclang.so.9
  lrwxrwxrwx 1 wink users   13 May 13 10:49 
build-clang-shlib-master/lib/libclang.so.9svn -> libclang.so.9

But the exported symbols are only the clang C api from libclang.exports:

  $ nm -g --demangle build-clang-shlib-master/lib/libclang.so.9 | grep ' T ' | 
wc -l
  381
  $ wc -l clang/tools/libclang/libclang.exports 
  381 clang/tools/libclang/libclang.exports

If I don't use your patch and build the same way lib/libclang.so.9 is still 
created and with the 381 globals:

  $ cmake ../llvm -G Ninja '-DLLVM_ENABLE_PROJECTS=clang;lld;compiler-rt' 
-DCMAKE_INSTALL_PREFIX=/home/wink/local-master -DCMAKE_BUILD_TYPE=Release 
-DLLVM_BUILD_LLVM_DYLIB=ON
  $ ninja -j11 -v
  $ ls build-master/lib/libclang*so* -al
  lrwxrwxrwx 1 wink users   16 May 13 11:35 build-master/lib/libclang.so -> 
libclang.so.9svn
  -rwxr-xr-x 1 wink users 81573360 May 13 11:35 build-master/lib/libclang.so.9
  lrwxrwxrwx 1 wink users   13 May 13 11:35 
build-master/lib/libclang.so.9svn -> libclang.so.9
  $ nm -g --demangle build-master/lib/libclang.so.9 | grep ' T ' | wc -l
  381

I was assuming the goal of clang-shlib was to export all of the symbols as 
exported by the libclang*.a files,
of so the first stab didn't quite do it.

But one thing it did do was reduce the steps and time ninja needed to complete 
my test
build from 4355 steps for 

[PATCH] D61804: Support building shared and static libraries simultaneously

[Chris Bieneman via Phabricator via cfe-commits]

beanz added a comment.

In D61804#1499275 , @winksaville wrote:

> When you say you don't think the build system should do this, what do you 
> mean?


`llvm_add_library` supports both, because there are places where we consciously 
choose to build both. That's not the problem. What I don't think the build 
system should do, is take an unsupported distribution format (which is 
unsupported for technical reasons), and make it supported without resolving the 
technical issues.

> With the change I've made when both are specified the default cmake entity, 
> ${name}, is the static libraries. When neither are specified the default is 
> static or shared depending on BUILD_SHARED_LIBS which is unchanged.

I missed that you were renaming the targets before calling `llvm_add_library` 
so that isn't an issue, but there are still lots of issues here. Like, if you 
specify your new option to generate clang static libraries but don't specify 
`BUILD_SHARED_LIBS` or `LLVM_USE_LLVM_DYLIB` each of your clang libraries will 
get statically linked against its own copy of LLVM, which not only makes your 
libraries big, it also makes them not work. You can't link more than one copy 
of LLVM into a single process memory space safely because of LLVM's use of 
global data.

> The problem is that some people want to use shared libraries and some want 
> static.

Why? I generally think most people don't understand the trade-offs, and I'm not 
sure we should feed into that.

> I'm trying to allow both to be built and ask the Arch Linux people to 
> distribute both.

Arch Linux is in an unsupported state, and your patch isn't the way forward.

Let's rewind.

Why do "some people like shared libraries"? There are usually two reasons 
people cite for linking shared libraries. One is reduced binary distribution 
size because you're reducing duplication of code. The other, is the ability to 
update libraries without updating tools.

The later, is not an issue here. LLVM and Clang's C++ libraries don't provide 
any API or ABI stability guarantees across releases, so there is no way you're 
going to reasonably update LLVM or Clang libraries without rebuilding the 
applications using them.

The former can be valuable, but it comes at a cost. Dynamic resolution of 
symbols slows down process launch time, and dynamic resolution of C++ link-once 
ODRs are exceptionally slow. If you are building a compiler, chances are you 
don't want to slow down your process launch that much. But let's pretend you 
don't care. Your solution still isn't the right way to do this.

Creating component shared libraries results in duplicating all the C++ 
link-once ODRs across each shared module. That will not only slow down your 
process launch, but bloat the size of your binary distribution.

In D61804#1499276 , @winksaville wrote:

> It looks to me that llvm, libunwind and libcxx support building both, so the 
> goal isn't unprecedented,


This is very much an apple and oranges comparison. For one, LLVM does not 
support generating component dylibs and shared libs. It supports generating 
libLLVM, a single dylib containing all of the LLVM components rolled together. 
libunwind, libcxx, libcxxabi, are runtime libraries that are designed to allow 
static linkage, and they don't link LLVM.

I apologize that I missed your thread on the dev list, because that would have 
been a much better place to have this conversation. Having gone back and read 
it now, it sounds to me like what you really want is a clang equivalent of 
libLLVM. That is a reasonable and viable thing to want. This patch 
(https://reviews.llvm.org/P8147) is a first stab. I'm happy to prepare that for 
landing in Clang if that meets your needs, and that is a viable way forward.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Tom Stellard via Phabricator via cfe-commits]

tstellar added a comment.

Not sure if this would be relevant for your use case, but it would be really 
nice to have a libCLANG.so with all the C++ symbols, like we do for llvm.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville updated this revision to Diff 199172.
winksaville added a comment.

Change default to be CLANG_ENABLE_STATIC_LIBRARIES=ON 
CLANG_ENABLE_SHARED_LIBRARIeS=OFF

Suggested by @beanz


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804

Files:
  clang/CMakeLists.txt
  clang/cmake/modules/AddClang.cmake
  llvm/cmake/modules/AddLLVM.cmake


Index: llvm/cmake/modules/AddLLVM.cmake
===
--- llvm/cmake/modules/AddLLVM.cmake
+++ llvm/cmake/modules/AddLLVM.cmake
@@ -442,20 +442,20 @@
   endif()
 
   if(ARG_SHARED AND ARG_STATIC)
-# static
-set(name_static "${name}_static")
+# Do shared first
+set(name_shared "${name}_shared")
 if(ARG_OUTPUT_NAME)
   set(output_name OUTPUT_NAME "${ARG_OUTPUT_NAME}")
 endif()
 # DEPENDS has been appended to LLVM_COMMON_LIBS.
-llvm_add_library(${name_static} STATIC
+llvm_add_library(${name_shared} SHARED
   ${output_name}
   OBJLIBS ${ALL_FILES} # objlib
   LINK_LIBS ${ARG_LINK_LIBS}
   LINK_COMPONENTS ${ARG_LINK_COMPONENTS}
   )
-# FIXME: Add name_static to anywhere in TARGET ${name}'s PROPERTY.
-set(ARG_STATIC)
+# FIXME: Add name_shared to anywhere in TARGET ${name}'s PROPERTY.
+set(ARG_SHARED)
   endif()
 
   if(ARG_MODULE)
Index: clang/cmake/modules/AddClang.cmake
===
--- clang/cmake/modules/AddClang.cmake
+++ clang/cmake/modules/AddClang.cmake
@@ -44,8 +44,8 @@
 
 macro(add_clang_library name)
   cmake_parse_arguments(ARG
-"SHARED"
-""
+"SHARED;STATIC"
+"OUTPUT_NAME"
 "ADDITIONAL_HEADERS"
 ${ARGN})
   set(srcs)
@@ -80,10 +80,20 @@
   ${ARG_ADDITIONAL_HEADERS} # It may contain unparsed unknown args.
   )
   endif()
-  if(ARG_SHARED)
+  if(ARG_SHARED OR CLANG_ENABLE_SHARED_LIBRARIES)
 set(ARG_ENABLE_SHARED SHARED)
   endif()
-  llvm_add_library(${name} ${ARG_ENABLE_SHARED} ${ARG_UNPARSED_ARGUMENTS} 
${srcs})
+  if(ARG_STATIC OR CLANG_ENABLE_STATIC_LIBRARIES)
+set(ARG_ENABLE_STATIC STATIC)
+  endif()
+  if(ARG_OUTPUT_NAME)
+set(NAME_OUTPUT OUTPUT_NAME ${ARG_OUTPUT_NAME})
+  elseif(ARG_SHARED AND ARG_STATIC)
+set(NAME_OUTPUT OUTPUT_NAME ${name})
+  else()
+set(NAME_OUTPUT)
+  endif()
+  llvm_add_library(${name} ${NAME_OUTPUT} ${ARG_ENABLE_SHARED} 
${ARG_ENABLE_STATIC} ${ARG_UNPARSED_ARGUMENTS} ${srcs})
 
   if(TARGET ${name})
 target_link_libraries(${name} INTERFACE ${LLVM_COMMON_LIBS})
Index: clang/CMakeLists.txt
===
--- clang/CMakeLists.txt
+++ clang/CMakeLists.txt
@@ -447,6 +447,9 @@
 option(CLANG_ENABLE_ARCMT "Build ARCMT." ON)
 option(CLANG_ENABLE_STATIC_ANALYZER "Build static analyzer." ON)
 
+option(CLANG_ENABLE_SHARED_LIBRARIES "Build libclang* as shared libraries." 
OFF)
+option(CLANG_ENABLE_STATIC_LIBRARIES "Build libclang* as static libraries." ON)
+
 option(CLANG_ENABLE_PROTO_FUZZER "Build Clang protobuf fuzzer." OFF)
 
 if(NOT CLANG_ENABLE_STATIC_ANALYZER AND CLANG_ENABLE_ARCMT)


Index: llvm/cmake/modules/AddLLVM.cmake
===
--- llvm/cmake/modules/AddLLVM.cmake
+++ llvm/cmake/modules/AddLLVM.cmake
@@ -442,20 +442,20 @@
   endif()
 
   if(ARG_SHARED AND ARG_STATIC)
-# static
-set(name_static "${name}_static")
+# Do shared first
+set(name_shared "${name}_shared")
 if(ARG_OUTPUT_NAME)
   set(output_name OUTPUT_NAME "${ARG_OUTPUT_NAME}")
 endif()
 # DEPENDS has been appended to LLVM_COMMON_LIBS.
-llvm_add_library(${name_static} STATIC
+llvm_add_library(${name_shared} SHARED
   ${output_name}
   OBJLIBS ${ALL_FILES} # objlib
   LINK_LIBS ${ARG_LINK_LIBS}
   LINK_COMPONENTS ${ARG_LINK_COMPONENTS}
   )
-# FIXME: Add name_static to anywhere in TARGET ${name}'s PROPERTY.
-set(ARG_STATIC)
+# FIXME: Add name_shared to anywhere in TARGET ${name}'s PROPERTY.
+set(ARG_SHARED)
   endif()
 
   if(ARG_MODULE)
Index: clang/cmake/modules/AddClang.cmake
===
--- clang/cmake/modules/AddClang.cmake
+++ clang/cmake/modules/AddClang.cmake
@@ -44,8 +44,8 @@
 
 macro(add_clang_library name)
   cmake_parse_arguments(ARG
-"SHARED"
-""
+"SHARED;STATIC"
+"OUTPUT_NAME"
 "ADDITIONAL_HEADERS"
 ${ARGN})
   set(srcs)
@@ -80,10 +80,20 @@
   ${ARG_ADDITIONAL_HEADERS} # It may contain unparsed unknown args.
   )
   endif()
-  if(ARG_SHARED)
+  if(ARG_SHARED OR CLANG_ENABLE_SHARED_LIBRARIES)
 set(ARG_ENABLE_SHARED SHARED)
   endif()
-  llvm_add_library(${name} ${ARG_ENABLE_SHARED} ${ARG_UNPARSED_ARGUMENTS} ${srcs})
+  if(ARG_STATIC OR CLANG_ENABLE_STATIC_LIBRARIES)
+set(ARG_ENABLE_STATIC STATIC)
+  endif()
+  if(ARG_OUTPUT_NAME)
+set(NAME_OUTPUT OUTPUT_NAME 

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville marked an inline comment as done.
winksaville added a comment.

In D61804#1499267 , @beanz wrote:

> As an additional note, Arch linux should not be building clang with 
> `BUILD_SHARED_LIBS` nor should it be distributing those ,so files. That isn't 
> a supported configuration for Clang deployment.


Agreed, but I'd like to give them a choice of making both if I can get this 
accepted. It looks to me that llvm, libunwind and libcxx support building both, 
so the goal isn't unprecedented,




Comment at: clang/CMakeLists.txt:451
+option(CLANG_ENABLE_SHARED_LIBRARIES "Build libclang* as shared libraries." ON)
+option(CLANG_ENABLE_STATIC_LIBRARIES "Build libclang* as static libraries." ON)
+

beanz wrote:
> These shouldn't both default to `On`, that is a change in behavior that would 
> be a build-time regression for anyone not interested in building shared 
> libraries. `STATIC` should default `On`, and `SHARED` default `Off`.
> 
> Also you need to check that one of the two options is enabled. If both are 
> `Off` confusing things will happen.
I'll change it to default SHARED OFF and STATIC ON.

When both are off it acts as it does without these changes, the default will 
depend on BUILD_SHARED_LIBS:
```
# llvm_add_library(name sources...
#   SHARED;STATIC
# STATIC by default w/o BUILD_SHARED_LIBS.
# SHARED by default w/  BUILD_SHARED_LIBS.
```



Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville added a comment.

In D61804#1499259 , @beanz wrote:

> I question the general utility of this patch, and I don't really think this 
> does anything we want the build system doing.


When you say you don't think the build system should do this, what do you mean?
The llvm_add_library routine without my change already supports generating both,
my change just changes the default cmake entities created from being:

  shared: ${name}
  static: ${name}_static

to:

  shared: ${name}_shared
  static: ${name}



> If you specify `STATIC` and `SHARED` to `llvm_add_library` it creates the 
> shared library target as the one that everything links against, which is 
> exactly what you don't want.

With the change I've made when both are specified the default cmake entity, 
${name}, is the static libraries. When neither are specified the default is 
static or shared depending on BUILD_SHARED_LIBS which is unchanged.

> There are two primary reasons why `BUILD_SHARED_LIBS` is not recommended for 
> non-development builds both have to do with how shared object linking works:
> 
> (1) Is that clang when linked against shared libraries is *way* slower (I 
> think last I measured was over 10%). This is caused by the need for resolving 
> linkage at process launch every time a process is launched.
> 
> (2) Is that LLVM relies on a strange mess of static initializers used for 
> static registration. The most problematic one is cl::opt. When you link a 
> static archive only the object files that contain unresolved symbols get 
> pulled in, when you link against a shared object, all of it gets pulled in. 
> Because static archives selectively pull in object files (and their static 
> initializers) we have had issues in the past where the static build of LLVM 
> works, but the shared build produces runtime errors from cl::opt.
> 
> I'm curious to understand the problem you are trying to solve. From other 
> posts on the list it really sounds to me like you want a variant of libClang 
> that exports the entire Clang API surface rather than just the C API. I think 
> there are better ways to accomplish that.

The problem is that some people want to use shared libraries and some want 
static. In particular Arch Linux prefers shared over static and with clang 
there is no option for building both so they end up building only shared. I'm 
trying to allow both to be built and ask the Arch Linux people to distribute 
both.

> Exposing Clang's libraries for piecemeal consumption is a much bigger problem 
> than just the build system not building libClang*.so. For example, Clang has 
> no equivalent of llvm-config, so there is no external representation of the 
> clang dependency graph that users can rely on.
> 
> This all aside, I also have issues with this implementation described inline 
> below.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Chris Bieneman via Phabricator via cfe-commits]

beanz added a comment.

As an additional note, Arch linux should not be building clang with 
`BUILD_SHARED_LIBS` nor should it be distributing those ,so files. That isn't a 
supported configuration for Clang deployment.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Chris Bieneman via Phabricator via cfe-commits]

beanz added a comment.

I question the general utility of this patch, and I don't really think this 
does anything we want the build system doing.

If you specify `STATIC` and `SHARED` to `llvm_add_library` it creates the 
shared library target as the one that everything links against, which is 
exactly what you don't want. There are two primary reasons why 
`BUILD_SHARED_LIBS` is not recommended for non-development builds both have to 
do with how shared object linking works:

(1) Is that clang when linked against shared libraries is *way* slower (I think 
last I measured was over 10%). This is caused by the need for resolving linkage 
at process launch every time a process is launched.

(2) Is that LLVM relies on a strange mess of static initializers used for 
static registration. The most problematic one is cl::opt. When you link a 
static archive only the object files that contain unresolved symbols get pulled 
in, when you link against a shared object, all of it gets pulled in. Because 
static archives selectively pull in object files (and their static 
initializers) we have had issues in the past where the static build of LLVM 
works, but the shared build produces runtime errors from cl::opt.

I'm curious to understand the problem you are trying to solve. From other posts 
on the list it really sounds to me like you want a variant of libClang that 
exports the entire Clang API surface rather than just the C API. I think there 
are better ways to accomplish that.

Exposing Clang's libraries for piecemeal consumption is a much bigger problem 
than just the build system not building libClang*.so. For example, Clang has no 
equivalent of llvm-config, so there is no external representation of the clang 
dependency graph that users can rely on.

This all aside, I also have issues with this implementation described inline 
below.




Comment at: clang/CMakeLists.txt:451
+option(CLANG_ENABLE_SHARED_LIBRARIES "Build libclang* as shared libraries." ON)
+option(CLANG_ENABLE_STATIC_LIBRARIES "Build libclang* as static libraries." ON)
+

These shouldn't both default to `On`, that is a change in behavior that would 
be a build-time regression for anyone not interested in building shared 
libraries. `STATIC` should default `On`, and `SHARED` default `Off`.

Also you need to check that one of the two options is enabled. If both are 
`Off` confusing things will happen.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D61804/new/

https://reviews.llvm.org/D61804



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D61804: Support building shared and static libraries simultaneously

[Wink Saville via Phabricator via cfe-commits]

winksaville created this revision.
winksaville added reviewers: tstellar, morehouse, gottesmm, chapuni.
Herald added subscribers: llvm-commits, cfe-commits, mgorny.
Herald added projects: clang, LLVM.

I ran into the need for both while trying to compile zig (https://ziglang.org)
on my Arch Linux system and it fails. The problem is that the Arch Linux
clang package only provides shared libraries using the following cmake
command:

  cmake .. -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=/usr \
-DPYTHON_EXECUTABLE=/usr/bin/python \
-DBUILD_SHARED_LIBS=ON \
-DLLVM_LINK_LLVM_DYLIB=ON \
-DLLVM_ENABLE_RTTI=ON \
-DLLVM_BUILD_TESTS=ON \
-DLLVM_INCLUDE_DOCS=ON \
-DLLVM_BUILD_DOCS=ON \
-DLLVM_ENABLE_SPHINX=ON \
-DSPHINX_WARNINGS_AS_ERRORS=OFF \
-DLLVM_EXTERNAL_LIT=/usr/bin/lit \
-DLLVM_MAIN_SRC_DIR="$srcdir/llvm-$pkgver.src"

In particular, it uses "BUILD_SHARED_LIBS=ON" to get the shared
libraries. This works but there are no static libraries and further
a note in the documentation says,
https://llvm.org/docs/CMake.html#llvm-specific-variables :

  "BUILD_SHARED_LIBS is only recommended for use by LLVM developers.
  If you want to build LLVM as a shared library, you should use the
  LLVM_BUILD_LLVM_DYLIB option."

So it seemed to me it a solution would be to provide an easy way to build
clang or other llvm subprojects with both static and shared libraries.

As it turns out on small number modifications were needed:

  clang/CMAkeLists.txt:
  
   - Add options CLANG_ENABLE_STATIC_LIBRARIES and CLANG_ENABLE_SHARED_LIBRARIES
 These are defaulted to ON so libclang* libraries are built as
 static and shared.  As these are options they can be overridden on the
 cmake command line.
  
  clang/cmake/modules/AddClang.cmake add_clang_library:
  
   - Add support for STATIC, SHARED or both
   - Add support for OUTPUT_NAME which defaults to the ${name} passed to
 add_clang_library
  
  llvm/cmake/modules/AddLLVM.cmake llvm_add_library:
  
   - Changed so when both SHARED and STATIC are passed SHARED is built
 first as cmake object ${name}_shared.
  
 Without this change llvm_add_library causes the shared libraries to
 be linked to clang rather than the static library and clang fails
 when run with:
  
   $ ./bin/clang-9 --version
   : CommandLine Error: Option 'use-dbg-addr' registered more than once!
   LLVM ERROR: inconsistency in registered CommandLine options


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D61804

Files:
  clang/CMakeLists.txt
  clang/cmake/modules/AddClang.cmake
  llvm/cmake/modules/AddLLVM.cmake


Index: llvm/cmake/modules/AddLLVM.cmake
===
--- llvm/cmake/modules/AddLLVM.cmake
+++ llvm/cmake/modules/AddLLVM.cmake
@@ -442,20 +442,20 @@
   endif()
 
   if(ARG_SHARED AND ARG_STATIC)
-# static
-set(name_static "${name}_static")
+# Do shared first
+set(name_shared "${name}_shared")
 if(ARG_OUTPUT_NAME)
   set(output_name OUTPUT_NAME "${ARG_OUTPUT_NAME}")
 endif()
 # DEPENDS has been appended to LLVM_COMMON_LIBS.
-llvm_add_library(${name_static} STATIC
+llvm_add_library(${name_shared} SHARED
   ${output_name}
   OBJLIBS ${ALL_FILES} # objlib
   LINK_LIBS ${ARG_LINK_LIBS}
   LINK_COMPONENTS ${ARG_LINK_COMPONENTS}
   )
-# FIXME: Add name_static to anywhere in TARGET ${name}'s PROPERTY.
-set(ARG_STATIC)
+# FIXME: Add name_shared to anywhere in TARGET ${name}'s PROPERTY.
+set(ARG_SHARED)
   endif()
 
   if(ARG_MODULE)
Index: clang/cmake/modules/AddClang.cmake
===
--- clang/cmake/modules/AddClang.cmake
+++ clang/cmake/modules/AddClang.cmake
@@ -44,8 +44,8 @@
 
 macro(add_clang_library name)
   cmake_parse_arguments(ARG
-"SHARED"
-""
+"SHARED;STATIC"
+"OUTPUT_NAME"
 "ADDITIONAL_HEADERS"
 ${ARGN})
   set(srcs)
@@ -80,10 +80,20 @@
   ${ARG_ADDITIONAL_HEADERS} # It may contain unparsed unknown args.
   )
   endif()
-  if(ARG_SHARED)
+  if(ARG_SHARED OR CLANG_ENABLE_SHARED_LIBRARIES)
 set(ARG_ENABLE_SHARED SHARED)
   endif()
-  llvm_add_library(${name} ${ARG_ENABLE_SHARED} ${ARG_UNPARSED_ARGUMENTS} 
${srcs})
+  if(ARG_STATIC OR CLANG_ENABLE_STATIC_LIBRARIES)
+set(ARG_ENABLE_STATIC STATIC)
+  endif()
+  if(ARG_OUTPUT_NAME)
+set(NAME_OUTPUT OUTPUT_NAME ${ARG_OUTPUT_NAME})
+  elseif(ARG_SHARED AND ARG_STATIC)
+set(NAME_OUTPUT OUTPUT_NAME ${name})
+  else()
+set(NAME_OUTPUT)
+  endif()
+  llvm_add_library(${name} ${NAME_OUTPUT} ${ARG_ENABLE_SHARED} 
${ARG_ENABLE_STATIC} ${ARG_UNPARSED_ARGUMENTS} ${srcs})
 
   if(TARGET ${name})
 target_link_libraries(${name} INTERFACE ${LLVM_COMMON_LIBS})
Index: clang/CMakeLists.txt
===
--- clang/CMakeLists.txt
+++ clang/CMakeLists.txt
@@