Author: yuripv
Date: Tue Nov 6 21:49:50 2018
New Revision: 340204
URL: https://svnweb.freebsd.org/changeset/base/340204
Log:
Cleanup locale tools:
- Simplify the source dir specification, and update README
appropriately
- Drop the LC (doonly) processing, it's broken, and even if fixed, not
really useful
- Don't remove the target directories while installing new data as it
removes Makefile.depend which we don't manage; only rm the files we
are going to add/replace/delete instead
- Restrict adding bsd.endian.mk to colldef and ctypedef Makefiles, it's
not needed in other (text-only) categories
- GC unused scripts; they don't seem to be particularly helpful standalone
as well
Reviewed by: bapt
Approved by: kib (mentor, implicit)
Differential Revision: https://reviews.freebsd.org/D17858
Deleted:
head/tools/tools/locale/etc/unicode.conf
head/tools/tools/locale/tools/UTF82encoding.pl
head/tools/tools/locale/tools/changeoffset.pl
head/tools/tools/locale/tools/unicode2src.pl
head/tools/tools/locale/tools/whatis.pl
Modified:
head/tools/tools/locale/Makefile
head/tools/tools/locale/README
head/tools/tools/locale/tools/cldr2def.pl
head/tools/tools/locale/tools/finalize
head/tools/tools/locale/tools/utf8-rollup.pl
Modified: head/tools/tools/locale/Makefile
==============================================================================
--- head/tools/tools/locale/Makefile Tue Nov 6 21:47:22 2018
(r340203)
+++ head/tools/tools/locale/Makefile Tue Nov 6 21:49:50 2018
(r340204)
@@ -10,13 +10,10 @@
.OBJDIR: .
-.if !defined(CLDRDIR)
-CLDRDIR!= grep ^cldr etc/unicode.conf | cut -f 2 -d " "
+.if !defined(UNIDIR)
+.error UNIDIR is not set
.endif
-.if !defined(UNIDATADIR)
-UNIDATADIR!= grep ^unidata etc/unicode.conf | cut -f 2 -d " "
-.endif
-PASSON= CLDRDIR="${CLDRDIR}" UNIDATADIR="${UNIDATADIR}"
+PASSON= UNIDIR="${UNIDIR}"
ETCDIR= ${.CURDIR}/etc
@@ -49,12 +46,8 @@ COLLATIONS_SPECIAL_ENV+= ${area}.${enc}
.endfor
PASSON+= COLLATIONS_SPECIAL="${COLLATIONS_SPECIAL_ENV}"
-.if defined(LC)
-LC:= --lc=${LC}
-.endif
-
all:
- cp ${ETCDIR}/common.UTF-8.src ${CLDRDIR}/posix/xx_Comm_C.UTF-8.src
+ cp ${ETCDIR}/common.UTF-8.src ${UNIDIR}/posix/xx_Comm_C.UTF-8.src
.for t in ${TYPES}
. if ${KNOWN:M${t}}
test -d ${t} || mkdir ${t}
@@ -69,8 +62,9 @@ install: install-${t}
install-${t}:
. if ${KNOWN:M${t}}
rm -rf ${.CURDIR}/${t}.draft
- rm -rf ${.CURDIR}/../../../share/${t}
- mv ${.CURDIR}/${t} ${.CURDIR}/../../../share/${t}
+ rm -f ${.CURDIR}/../../../share/${t}/Makefile
+ rm -f ${.CURDIR}/../../../share/${t}/*.src
+ mv ${.CURDIR}/${t}/* ${.CURDIR}/../../../share/${t}/
. endif
.endfor
@@ -86,10 +80,9 @@ post-install:
gen-${t}:
mkdir -p ${t} ${t}.draft
perl -I tools tools/cldr2def.pl \
- --cldr=$$(realpath ${CLDRDIR}) \
- --unidata=$$(realpath ${UNIDATADIR}) \
+ --unidir=$$(realpath ${UNIDIR}) \
--etc=$$(realpath ${ETCDIR}) \
- --type=${t} ${LC}
+ --type=${t}
build-${t}: gen-${t}
env ${PASSON} tools/finalize ${t}
@@ -101,15 +94,16 @@ build-colldef: static-colldef
static-colldef:
.for area enc in ${COLLATION_SPECIAL}
- awk -f tools/extract-colldef.awk ${CLDRDIR}/posix/${area}.${enc}.src >
colldef.draft/${area}.${enc}.src
+ awk -f tools/extract-colldef.awk ${UNIDIR}/posix/${area}.${enc}.src > \
+ colldef.draft/${area}.${enc}.src
.endfor
transfer-rollup:
- cp ${ETCDIR}/common.UTF-8.src ${CLDRDIR}/posix/xx_Comm_C.UTF-8.src
+ cp ${ETCDIR}/common.UTF-8.src ${UNIDIR}/posix/xx_Comm_C.UTF-8.src
rollup:
perl -I tools tools/utf8-rollup.pl \
- --cldr=$$(realpath ${CLDRDIR}) \
+ --unidir=$$(realpath ${UNIDIR}) \
--etc=$$(realpath ${ETCDIR})
clean:
@@ -159,33 +153,33 @@ ENCODINGS= Big5 \
POSIX:
-.if exists (${CLDRDIR}/tools/java/cldr.jar)
- mkdir -p ${CLDRDIR}/posix
+.if exists (${UNIDIR}/tools/java/cldr.jar)
+ mkdir -p ${UNIDIR}/posix
. for area in ${BASE_LOCALES_OF_INTEREST}
-. if !exists(${CLDRDIR}/posix/${area}.UTF-8.src)
- java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \
+. if !exists(${UNIDIR}/posix/${area}.UTF-8.src)
+ java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \
org.unicode.cldr.posix.GeneratePOSIX \
- -d ${CLDRDIR}/posix -m ${area} -c UTF-8
+ -d ${UNIDIR}/posix -m ${area} -c UTF-8
. endif
. endfor
. for area encoding in ${COLLATION_SPECIAL}
-. if !exists(${CLDRDIR}/posix/${area}.${encoding}.src)
- java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \
+. if !exists(${UNIDIR}/posix/${area}.${encoding}.src)
+ java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \
org.unicode.cldr.posix.GeneratePOSIX \
- -d ${CLDRDIR}/posix -m ${area} -c ${encoding}
+ -d ${UNIDIR}/posix -m ${area} -c ${encoding}
. endif
. endfor
. for enc in ${ENCODINGS}
-. if !exists(${CLDRDIR}/posix/${enc}.cm)
- java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \
+. if !exists(${UNIDIR}/posix/${enc}.cm)
+ java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \
org.unicode.cldr.posix.GenerateCharmap \
- -d ${CLDRDIR}/posix -c ${enc}
+ -d ${UNIDIR}/posix -c ${enc}
. endif
. endfor
.else
@echo "Please install CLDR toolset for the desired release"
- @echo "It should go at ${CLDRDIR}/tools"
+ @echo "It should go at ${UNIDIR}/tools"
.endif
clean-POSIX:
- rm -f ${CLDRDIR}/posix/*
+ rm -f ${UNIDIR}/posix/*
Modified: head/tools/tools/locale/README
==============================================================================
--- head/tools/tools/locale/README Tue Nov 6 21:47:22 2018
(r340203)
+++ head/tools/tools/locale/README Tue Nov 6 21:49:50 2018
(r340204)
@@ -9,30 +9,23 @@ Tools needed:
devel/p5-Tie-IxHash
textproc/p5-XML-Parser
-Fetch CLDR data from: http://unicode.org/Public/cldr/. You need all of the
+1. Fetch CLDR data from: http://unicode.org/Public/cldr/. You need all of the
core.zip, keyboards.zip, and tools.zip.
-
-Extract:
- mkdir -p ~/unicode/cldr/v33.0
- cd ~/unicode/cldr/v33.0
- unzip ~/core.zip ~/keyboards.zip ~/tools.zip
-
-Fetch unidata (UCD.zip) from http://www.unicode.org/Public/zipped/latest.
-
-Extract:
- mkdir -p ~/unicode/UNIDATA/11.0.0
- cd ~/unicode/UNIDATA/11.0.0
+2. Fetch unidata (UCD.zip) from http://www.unicode.org/Public/zipped/latest.
+3. Extract:
+ mkdir -p ~/unicode
+ cd ~/unicode
+ unzip ~/core.zip
+ unzip ~/keyboards.zip
+ unzip ~/tools.zip
unzip ~/UCD.zip
-
-Either modify tools/tools/locales/etc/unicode.conf or export variables:
- CLDRDIR=~/unicode/cldr/v33.0; export CLDRDIR
- UNIDATADIR=~/unicode/UNIDATA/9.0.0; export UNIDATADIR
-
-Build the CLDR tools:
- cd $CLDRDIR/tools/java
+4. Export variable:
+ UNIDIR=~/unicode; export UNIDIR
+5. Build the CLDR tools:
+ cd $UNIDIR/tools/java
ant jar
-
-Run:
+6. Build POSIX data files from CLDR data:
make POSIX
+7. Build and install new locale data:
make
make install
Modified: head/tools/tools/locale/tools/cldr2def.pl
==============================================================================
--- head/tools/tools/locale/tools/cldr2def.pl Tue Nov 6 21:47:22 2018
(r340203)
+++ head/tools/tools/locale/tools/cldr2def.pl Tue Nov 6 21:49:50 2018
(r340204)
@@ -6,32 +6,27 @@ use File::Copy;
use XML::Parser;
use Tie::IxHash;
use Text::Iconv;
-use Data::Dumper;
+#use Data::Dumper;
use Getopt::Long;
use Digest::SHA qw(sha1_hex);
require "charmaps.pm";
if ($#ARGV < 2) {
- print "Usage: $0 --cldr=<cldrdir> --unidata=<unidatadir> --etc=<etcdir>
--type=<type> [--lc=<la_CC>]\n";
+ print "Usage: $0 --unidir=<unidir> --etc=<etcdir> --type=<type>\n";
exit(1);
}
my $DEFENCODING = "UTF-8";
-my @filter = ();
-my $CLDRDIR = undef;
-my $UNIDATADIR = undef;
+my $UNIDIR = undef;
my $ETCDIR = undef;
my $TYPE = undef;
-my $doonly = undef;
my $result = GetOptions (
- "cldr=s" => \$CLDRDIR,
- "unidata=s" => \$UNIDATADIR,
+ "unidir=s" => \$UNIDIR,
"etc=s" => \$ETCDIR,
"type=s" => \$TYPE,
- "lc=s" => \$doonly
);
my %convertors = ();
@@ -47,8 +42,8 @@ get_languages();
my %utf8map = ();
my %utf8aliases = ();
-get_unidata($UNIDATADIR);
-get_utf8map("$CLDRDIR/posix/$DEFENCODING.cm");
+get_unidata($UNIDIR);
+get_utf8map("$UNIDIR/posix/$DEFENCODING.cm");
get_encodings("$ETCDIR/charmaps");
my %keys = ();
@@ -397,22 +392,6 @@ sub get_languages {
%translations = %{$data{T}};
%alternativemonths = %{$data{AM}};
%encodings = %{$data{E}};
-
- return if (!defined $doonly);
-
- my @a = split(/_/, $doonly);
- if ($#a == 1) {
- $filter[0] = $a[0];
- $filter[1] = "x";
- $filter[2] = $a[1];
- } elsif ($#a == 2) {
- $filter[0] = $a[0];
- $filter[1] = $a[1];
- $filter[2] = $a[2];
- }
-
- print Dumper(@filter);
- return;
}
sub transform_ctypes {
@@ -422,8 +401,6 @@ sub transform_ctypes {
foreach my $l (sort keys(%languages)) {
foreach my $f (sort keys(%{$languages{$l}})) {
foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
- next if ($#filter == 2 && ($filter[0] ne $l
- || $filter[1] ne $f || $filter[2] ne $c));
next if (defined $languages{$l}{$f}{definitions}
&& $languages{$l}{$f}{definitions} !~ /$TYPE/);
$languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread
@@ -432,7 +409,7 @@ sub transform_ctypes {
$file .= "_" . $c if ($c ne "x");
my $actfile = $file;
- my $filename = "$CLDRDIR/posix/xx_Comm_C.UTF-8.src";
+ my $filename = "$UNIDIR/posix/xx_Comm_C.UTF-8.src";
if (! -f $filename) {
print STDERR "Cannot open $filename\n";
next;
@@ -455,7 +432,7 @@ sub transform_ctypes {
close(FOUT);
foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) {
next if ($enc eq $DEFENCODING);
- $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+ $filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
if (! -f $filename) {
print STDERR "Cannot open $filename\n";
next;
@@ -494,8 +471,6 @@ sub transform_collation {
foreach my $l (sort keys(%languages)) {
foreach my $f (sort keys(%{$languages{$l}})) {
foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
- next if ($#filter == 2 && ($filter[0] ne $l
- || $filter[1] ne $f || $filter[2] ne $c));
next if (defined $languages{$l}{$f}{definitions}
&& $languages{$l}{$f}{definitions} !~ /$TYPE/);
$languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread
@@ -505,15 +480,15 @@ sub transform_collation {
$file .= $c;
my $actfile = $file;
- my $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+ my $filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
$filename = "$ETCDIR/$file.$DEFENCODING.src"
if (! -f $filename);
if (! -f $filename
&& defined $languages{$l}{$f}{fallback}) {
$file = $languages{$l}{$f}{fallback};
- $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+ $filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
}
- $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"
+ $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"
if (! -f $filename);
if (! -f $filename) {
print STDERR
@@ -564,8 +539,6 @@ sub get_fields {
foreach my $l (sort keys(%languages)) {
foreach my $f (sort keys(%{$languages{$l}})) {
foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
- next if ($#filter == 2 && ($filter[0] ne $l
- || $filter[1] ne $f || $filter[2] ne $c));
next if (defined $languages{$l}{$f}{definitions}
&& $languages{$l}{$f}{definitions} !~ /$TYPE/);
@@ -575,15 +548,15 @@ sub get_fields {
$file .= $f . "_" if ($f ne "x");
$file .= $c;
- my $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+ my $filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
$filename = "$ETCDIR/$file.$DEFENCODING.src"
if (! -f $filename);
if (! -f $filename
&& defined $languages{$l}{$f}{fallback}) {
$file = $languages{$l}{$f}{fallback};
- $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+ $filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
}
- $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"
+ $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"
if (! -f $filename);
if (! -f $filename) {
print STDERR
@@ -703,8 +676,6 @@ sub print_fields {
foreach my $l (sort keys(%languages)) {
foreach my $f (sort keys(%{$languages{$l}})) {
foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
- next if ($#filter == 2 && ($filter[0] ne $l
- || $filter[1] ne $f || $filter[2] ne $c));
next if (defined $languages{$l}{$f}{definitions}
&& $languages{$l}{$f}{definitions} !~ /$TYPE/);
foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) {
@@ -851,7 +822,6 @@ EOF
}
sub make_makefile {
- return if ($#filter > -1);
print "Creating Makefile for $TYPE\n";
my $SRCOUT;
my $SRCOUT2;
@@ -913,8 +883,16 @@ LOCALEDIR= \${SHAREDIR}/locale
FILESNAME= $FILESNAMES{$TYPE}
.SUFFIXES: .src .${SRCOUT2}
${MAPLOC}
+EOF
+
+ if ($TYPE eq "colldef" || $TYPE eq "ctypedef") {
+ print FOUT <<EOF;
.include <bsd.endian.mk>
+EOF
+ }
+
+ print FOUT <<EOF;
.src.${SRCOUT2}:
$SRCOUT
@@ -979,8 +957,6 @@ EOF
foreach my $l (sort keys(%languages)) {
foreach my $f (sort keys(%{$languages{$l}})) {
foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
- next if ($#filter == 2 && ($filter[0] ne $l
- || $filter[1] ne $f || $filter[2] ne $c));
next if (defined $languages{$l}{$f}{definitions}
&& $languages{$l}{$f}{definitions} !~ /$TYPE/);
if (defined $languages{$l}{$f}{data}{$c}{$DEFENCODING}
Modified: head/tools/tools/locale/tools/finalize
==============================================================================
--- head/tools/tools/locale/tools/finalize Tue Nov 6 21:47:22 2018
(r340203)
+++ head/tools/tools/locale/tools/finalize Tue Nov 6 21:49:50 2018
(r340204)
@@ -116,13 +116,13 @@ then
else {last1 = $1; last2 = $2}}' ${TEMP2} > ${TEMP3}
rm -f ${TEMP2}
/usr/bin/sed -E -e 's/[ ]+/ /g' \
- ${CLDRDIR}/posix/UTF-8.cm \
+ ${UNIDIR}/posix/UTF-8.cm \
> ${base}/../etc/final-maps/map.UTF-8
/usr/bin/sed -E -e 's/[ ]+/ /g' \
- ${CLDRDIR}/posix/eucCN.cm \
+ ${UNIDIR}/posix/eucCN.cm \
> ${base}/../etc/final-maps/map.eucCN
/usr/bin/sed -E -e 's/[ ]+/ /g' \
- ${CLDRDIR}/posix/eucCN.cm \
+ ${UNIDIR}/posix/eucCN.cm \
> ${base}/../etc/final-maps/map.GB2312
CHARMAPS="ARMSCII-8 Big5 CP1131 CP1251 \
CP866 GBK ISCII-DEV ISO8859-1 \
Modified: head/tools/tools/locale/tools/utf8-rollup.pl
==============================================================================
--- head/tools/tools/locale/tools/utf8-rollup.pl Tue Nov 6 21:47:22
2018 (r340203)
+++ head/tools/tools/locale/tools/utf8-rollup.pl Tue Nov 6 21:49:50
2018 (r340204)
@@ -1,4 +1,5 @@
#!/usr/local/bin/perl -wC
+# $FreeBSD$
use strict;
#use File::Copy;
@@ -11,15 +12,15 @@ use Getopt::Long;
if ($#ARGV != 1) {
- print "Usage: $0 --cldr=<cldrdir> --etc=<etcdir>\n";
+ print "Usage: $0 --unidir=<unidir> --etc=<etcdir>\n";
exit(1);
}
-my $CLDRDIR = undef;
+my $UNIDIR = undef;
my $ETCDIR = undef;
my $result = GetOptions (
- "cldr=s" => \$CLDRDIR,
+ "unidir=s" => \$UNIDIR,
"etc=s" => \$ETCDIR,
);
@@ -118,7 +119,7 @@ my $outfilename = "$ETCDIR/common.UTF-8.src";
my $manual_file = "$ETCDIR/manual-input.UTF-8";
my $stars =
"**********************************************************************\n";
-get_utf8map("$CLDRDIR/posix/UTF-8.cm");
+get_utf8map("$UNIDIR/posix/UTF-8.cm");
generate_header ();
generate_sections ();
generate_footer ();
@@ -252,7 +253,7 @@ sub compress_ctype {
my @lines = initialize_lines ($territory);
- my $filename = "$CLDRDIR/posix/$territory.UTF-8.src";
+ my $filename = "$UNIDIR/posix/$territory.UTF-8.src";
if (! -f $filename) {
print STDERR "Cannot open $filename\n";
return;
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
