ATLAS-117 Build fails on the latest commit ([email protected] via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/e77d3d33 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/e77d3d33 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/e77d3d33 Branch: refs/heads/master Commit: e77d3d336649560c70d574964a79be837e342bdb Parents: 133690f Author: Shwetha GS <[email protected]> Authored: Thu Sep 10 12:55:24 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Thu Sep 10 12:55:24 2015 +0530 ---------------------------------------------------------------------- distro/pom.xml | 120 + distro/src/bin/atlas_config.py | 291 +++ distro/src/bin/atlas_start.py | 81 + distro/src/bin/atlas_stop.py | 59 + distro/src/bin/cputil.py | 58 + distro/src/bin/quick_start.py | 66 + distro/src/conf/application.properties | 72 + distro/src/conf/atlas-env.sh | 49 + distro/src/conf/atlas-log4j.xml | 64 + distro/src/conf/client.properties | 38 + distro/src/conf/solr/currency.xml | 67 + distro/src/conf/solr/lang/stopwords_en.txt | 54 + distro/src/conf/solr/protwords.txt | 21 + distro/src/conf/solr/schema.xml | 534 ++++ distro/src/conf/solr/solrconfig.xml | 625 +++++ distro/src/conf/solr/stopwords.txt | 14 + distro/src/conf/solr/synonyms.txt | 29 + distro/src/main/assemblies/src-package.xml | 48 + .../src/main/assemblies/standalone-package.xml | 111 + distro/src/test/mock/LICENSE.txt | 26 + distro/src/test/mock/MANIFEST.in | 2 + distro/src/test/mock/README.txt | 179 ++ distro/src/test/mock/__init__.py | 1 + distro/src/test/mock/extendmock.py | 1 + distro/src/test/mock/mock.py | 2365 ++++++++++++++++++ distro/src/test/mock/mock.wpr | 26 + distro/src/test/mock/setup.cfg | 5 + distro/src/test/mock/setup.py | 72 + distro/src/test/mock/tests/__init__.py | 3 + distro/src/test/mock/tests/_testwith.py | 181 ++ distro/src/test/mock/tests/support.py | 41 + distro/src/test/mock/tests/support_with.py | 93 + distro/src/test/mock/tests/testcallable.py | 158 ++ distro/src/test/mock/tests/testhelpers.py | 940 +++++++ distro/src/test/mock/tests/testmagicmethods.py | 486 ++++ distro/src/test/mock/tests/testmock.py | 1351 ++++++++++ distro/src/test/mock/tests/testpatch.py | 1815 ++++++++++++++ distro/src/test/mock/tests/testsentinel.py | 33 + distro/src/test/mock/tests/testwith.py | 16 + distro/src/test/mock/tox.ini | 40 + distro/src/test/mock/unittest.cfg | 95 + distro/src/test/python/scripts/TestMetadata.py | 88 + distro/src/test/python/unitTests.py | 118 + docs/src/site/twiki/InstallationSteps.twiki | 4 +- pom.xml | 75 +- release-log.txt | 5 + src/bin/atlas_config.py | 291 --- src/bin/atlas_start.py | 81 - src/bin/atlas_stop.py | 59 - src/bin/cputil.py | 58 - src/bin/quick_start.py | 66 - src/conf/application.properties | 72 - src/conf/atlas-env.sh | 49 - src/conf/atlas-log4j.xml | 64 - src/conf/client.properties | 38 - src/conf/solr/currency.xml | 67 - src/conf/solr/lang/stopwords_en.txt | 54 - src/conf/solr/protwords.txt | 21 - src/conf/solr/schema.xml | 534 ---- src/conf/solr/solrconfig.xml | 625 ----- src/conf/solr/stopwords.txt | 14 - src/conf/solr/synonyms.txt | 29 - src/main/assemblies/src-package.xml | 48 - src/main/assemblies/standalone-package.xml | 111 - src/test/mock/LICENSE.txt | 26 - src/test/mock/MANIFEST.in | 2 - src/test/mock/README.txt | 179 -- src/test/mock/__init__.py | 1 - src/test/mock/extendmock.py | 1 - src/test/mock/mock.py | 2365 ------------------ src/test/mock/mock.wpr | 26 - src/test/mock/setup.cfg | 5 - src/test/mock/setup.py | 72 - src/test/mock/tests/__init__.py | 3 - src/test/mock/tests/_testwith.py | 181 -- src/test/mock/tests/support.py | 41 - src/test/mock/tests/support_with.py | 93 - src/test/mock/tests/testcallable.py | 158 -- src/test/mock/tests/testhelpers.py | 940 ------- src/test/mock/tests/testmagicmethods.py | 486 ---- src/test/mock/tests/testmock.py | 1351 ---------- src/test/mock/tests/testpatch.py | 1815 -------------- src/test/mock/tests/testsentinel.py | 33 - src/test/mock/tests/testwith.py | 16 - src/test/mock/tox.ini | 40 - src/test/mock/unittest.cfg | 95 - src/test/python/scripts/TestMetadata.py | 88 - src/test/python/unitTests.py | 118 - 88 files changed, 10544 insertions(+), 10492 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/pom.xml ---------------------------------------------------------------------- diff --git a/distro/pom.xml b/distro/pom.xml new file mode 100644 index 0000000..d42d78f --- /dev/null +++ b/distro/pom.xml @@ -0,0 +1,120 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>apache-atlas</artifactId> + <groupId>org.apache.atlas</groupId> + <version>0.6-incubating-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <artifactId>atlas-distro</artifactId> + <description>Apache Atlas Distribution</description> + <name>Apache Atlas Distribution</name> + + <profiles> + <profile> + <id>Windows</id> + <activation> + <os> + <family>windows</family> + </os> + </activation> + <properties> + <python.path.l> + ${project.basedir}\src\bin;${project.basedir}\src\test\python\scripts;${project.basedir}\src\test\mock + </python.path.l> + </properties> + </profile> + + <profile> + <id>Linux</id> + <activation> + <os> + <family>!windows</family> + </os> + </activation> + <properties> + <python.path.l> + ${project.basedir}/src/bin:${project.basedir}/src/test/mock:${project.basedir}/src/test/python/scripts + </python.path.l> + </properties> + </profile> + </profiles> + + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.2.1</version> + <inherited>false</inherited> + <executions> + <execution> + <configuration> + <executable>python</executable> + <workingDirectory>src/test/python</workingDirectory> + <arguments> + <argument>unitTests.py</argument> + </arguments> + <environmentVariables> + <PYTHONPATH>${python.path.l}</PYTHONPATH> + </environmentVariables> + </configuration> + <id>python-test</id> + <phase>test</phase> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <inherited>false</inherited> + <configuration> + <descriptors> + <descriptor>src/main/assemblies/standalone-package.xml</descriptor> + <descriptor>src/main/assemblies/src-package.xml</descriptor> + </descriptors> + <finalName>apache-atlas-${project.version}</finalName> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <!--Add a dependency here to ensure this module is the final one built--> + <dependencies> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>hive-bridge</artifactId> + <version>0.6-incubating-SNAPSHOT</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/bin/atlas_config.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py new file mode 100755 index 0000000..a3a23ef --- /dev/null +++ b/distro/src/bin/atlas_config.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import getpass + +import os +import platform +import subprocess +from threading import Thread +import sys +import time +import errno + +LIB = "lib" +CONF = "conf" +LOG="logs" +WEBAPP="server" + os.sep + "webapp" +DATA="data" +ENV_KEYS = ["JAVA_HOME", "METADATA_OPTS", "METADATA_LOG_DIR", "METADATA_PID_DIR", "METADATA_CONF", "METADATACPPATH", "METADATA_DATA_DIR", "METADATA_HOME_DIR", "METADATA_EXPANDED_WEBAPP_DIR"] +METADATA_CONF = "METADATA_CONF" +METADATA_LOG = "METADATA_LOG_DIR" +METADATA_PID = "METADATA_PID_DIR" +METADATA_WEBAPP = "METADATA_EXPANDED_WEBAPP_DIR" +METADATA_OPTS = "METADATA_OPTS" +METADATA_DATA = "METADATA_DATA_DIR" +METADATA_HOME = "METADATA_HOME_DIR" +IS_WINDOWS = platform.system() == "Windows" +ON_POSIX = 'posix' in sys.builtin_module_names +DEBUG = False + +def scriptDir(): + """ + get the script path + """ + return os.path.dirname(os.path.realpath(__file__)) + +def metadataDir(): + home = os.path.dirname(scriptDir()) + return os.environ.get(METADATA_HOME, home) + +def libDir(dir) : + return os.path.join(dir, LIB) + +def confDir(dir): + localconf = os.path.join(dir, CONF) + return os.environ.get(METADATA_CONF, localconf) + +def logDir(dir): + localLog = os.path.join(dir, LOG) + return os.environ.get(METADATA_LOG, localLog) + +def pidFile(dir): + localPid = os.path.join(dir, LOG) + return os.path.join(os.environ.get(METADATA_PID, localPid), 'atlas.pid') + +def dataDir(dir): + data = os.path.join(dir, DATA) + return os.environ.get(METADATA_DATA, data) + +def webAppDir(dir): + webapp = os.path.join(dir, WEBAPP) + return os.environ.get(METADATA_WEBAPP, webapp) + +def expandWebApp(dir): + webappDir = webAppDir(dir) + webAppMetadataDir = os.path.join(webappDir, "atlas") + d = os.sep + if not os.path.exists(os.path.join(webAppMetadataDir, "WEB-INF")): + try: + os.makedirs(webAppMetadataDir) + except OSError, e: + if e.errno != errno.EEXIST: + raise e + pass + os.chdir(webAppMetadataDir) + jar(os.path.join(metadataDir(), "server", "webapp", "atlas.war")) + +def dirMustExist(dirname): + if not os.path.exists(dirname): + os.mkdir(dirname) + return dirname + +def executeEnvSh(confDir): + envscript = '%s/atlas-env.sh' % confDir + if not IS_WINDOWS and os.path.exists(envscript): + envCmd = 'source %s && env' % envscript + command = ['bash', '-c', envCmd] + + proc = subprocess.Popen(command, stdout = subprocess.PIPE) + + for line in proc.stdout: + (key, _, value) = line.strip().partition("=") + if key in ENV_KEYS: + os.environ[key] = value + + proc.communicate() + +def java(classname, args, classpath, jvm_opts_list, logdir=None): + java_home = os.environ.get("JAVA_HOME", None) + if java_home: + prg = os.path.join(java_home, "bin", "java") + else: + prg = which("java") + + if prg is None: + raise EnvironmentError('The java binary could not be found in your path or JAVA_HOME') + + commandline = [prg] + commandline.extend(jvm_opts_list) + commandline.append("-classpath") + commandline.append(classpath) + commandline.append(classname) + commandline.extend(args) + return runProcess(commandline, logdir) + +def jar(path): + java_home = os.environ.get("JAVA_HOME", None) + if java_home: + prg = os.path.join(java_home, "bin", "jar") + else: + prg = which("jar") + + if prg is None: + raise EnvironmentError('The jar binary could not be found in your path or JAVA_HOME') + + commandline = [prg] + commandline.append("-xf") + commandline.append(path) + process = runProcess(commandline) + process.wait() + +def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + +def which(program): + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + return None + +def runProcess(commandline, logdir=None): + """ + Run a process + :param commandline: command line + :return:the return code + """ + global finished + debug ("Executing : %s" % commandline) + timestr = time.strftime("metadata.%Y%m%d-%H%M%S") + stdoutFile = None + stderrFile = None + if logdir: + stdoutFile = open(os.path.join(logdir, timestr + ".out"), "w") + stderrFile = open(os.path.join(logdir,timestr + ".err"), "w") + return subprocess.Popen(commandline, stdout=stdoutFile, stderr=stderrFile) + +def print_output(name, src, toStdErr): + """ + Relay the output stream to stdout line by line + :param name: + :param src: source stream + :param toStdErr: flag set if stderr is to be the dest + :return: + """ + + global needPassword + debug ("starting printer for %s" % name ) + line = "" + while not finished: + (line, done) = read(src, line) + if done: + out(toStdErr, line + "\n") + flush(toStdErr) + if line.find("Enter password for") >= 0: + needPassword = True + line = "" + out(toStdErr, line) + # closedown: read remainder of stream + c = src.read(1) + while c!="" : + c = c.decode('utf-8') + out(toStdErr, c) + if c == "\n": + flush(toStdErr) + c = src.read(1) + flush(toStdErr) + src.close() + +def read_input(name, exe): + """ + Read input from stdin and send to process + :param name: + :param process: process to send input to + :return: + """ + global needPassword + debug ("starting reader for %s" % name ) + while not finished: + if needPassword: + needPassword = False + if sys.stdin.isatty(): + cred = getpass.getpass() + else: + cred = sys.stdin.readline().rstrip() + exe.stdin.write(cred + "\n") + +def debug(text): + if DEBUG: print '[DEBUG] ' + text + + +def error(text): + print '[ERROR] ' + text + sys.stdout.flush() + +def info(text): + print text + sys.stdout.flush() + + +def out(toStdErr, text) : + """ + Write to one of the system output channels. + This action does not add newlines. If you want that: write them yourself + :param toStdErr: flag set if stderr is to be the dest + :param text: text to write. + :return: + """ + if toStdErr: + sys.stderr.write(text) + else: + sys.stdout.write(text) + +def flush(toStdErr) : + """ + Flush the output stream + :param toStdErr: flag set if stderr is to be the dest + :return: + """ + if toStdErr: + sys.stderr.flush() + else: + sys.stdout.flush() + +def read(pipe, line): + """ + read a char, append to the listing if there is a char that is not \n + :param pipe: pipe to read from + :param line: line being built up + :return: (the potentially updated line, flag indicating newline reached) + """ + + c = pipe.read(1) + if c != "": + o = c.decode('utf-8') + if o != '\n': + line += o + return line, False + else: + return line, True + else: + return line, False + +def writePid(metadata_pid_file, process): + f = open(metadata_pid_file, 'w') + f.write(str(process.pid)) + f.close() + + http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/bin/atlas_start.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py new file mode 100755 index 0000000..7680f04 --- /dev/null +++ b/distro/src/bin/atlas_start.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +import traceback + +import atlas_config as mc + +METADATA_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=application.log" +METADATA_COMMAND_OPTS="-Datlas.home=%s" +METADATA_CONFIG_OPTS="-Datlas.conf=%s" +DEFAULT_JVM_OPTS="-Xmx1024m -Dlog4j.configuration=atlas-log4j.xml" + +def main(): + + metadata_home = mc.metadataDir() + confdir = mc.dirMustExist(mc.confDir(metadata_home)) + mc.executeEnvSh(confdir) + logdir = mc.dirMustExist(mc.logDir(metadata_home)) + + #create sys property for conf dirs + jvm_opts_list = (METADATA_LOG_OPTS % logdir).split() + + cmd_opts = (METADATA_COMMAND_OPTS % metadata_home) + jvm_opts_list.extend(cmd_opts.split()) + + config_opts = (METADATA_CONFIG_OPTS % confdir) + jvm_opts_list.extend(config_opts.split()) + + default_jvm_opts = DEFAULT_JVM_OPTS + metadata_jvm_opts = os.environ.get(mc.METADATA_OPTS, default_jvm_opts) + jvm_opts_list.extend(metadata_jvm_opts.split()) + + #expand web app dir + web_app_dir = mc.webAppDir(metadata_home) + mc.expandWebApp(metadata_home) + + p = os.pathsep + metadata_classpath = confdir + p \ + + os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \ + + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "*" ) + p \ + + os.path.join(metadata_home, "libext", "*") + + metadata_pid_file = mc.pidFile(metadata_home) + + if os.path.isfile(metadata_pid_file): + print "%s already exists, exiting" % metadata_pid_file + sys.exit() + + args = ["-app", os.path.join(web_app_dir, "atlas")] + args.extend(sys.argv[1:]) + + process = mc.java("org.apache.atlas.Main", args, metadata_classpath, jvm_opts_list, logdir) + mc.writePid(metadata_pid_file, process) + + print "Apache Atlas Server started!!!\n" + +if __name__ == '__main__': + try: + returncode = main() + except Exception as e: + print "Exception: %s " % str(e) + print traceback.format_exc() + returncode = -1 + + sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/bin/atlas_stop.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_stop.py b/distro/src/bin/atlas_stop.py new file mode 100755 index 0000000..a157427 --- /dev/null +++ b/distro/src/bin/atlas_stop.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from signal import SIGTERM +import sys +import traceback + +import atlas_config as mc + +def main(): + + metadata_home = mc.metadataDir() + confdir = mc.dirMustExist(mc.confDir(metadata_home)) + mc.executeEnvSh(confdir) + piddir = mc.dirMustExist(mc.logDir(metadata_home)) + + metadata_pid_file = mc.pidFile(metadata_home) + + try: + pf = file(metadata_pid_file, 'r') + pid = int(pf.read().strip()) + pf.close() + except: + pid = None + + if not pid: + sys.stderr.write("No process ID file found. Server not running?\n") + return + + os.kill(pid, SIGTERM) + + # assuming kill worked since process check on windows is more involved... + if os.path.exists(metadata_pid_file): + os.remove(metadata_pid_file) + +if __name__ == '__main__': + try: + returncode = main() + except Exception as e: + print "Exception: %s " % str(e) + print traceback.format_exc() + returncode = -1 + + sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/bin/cputil.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/cputil.py b/distro/src/bin/cputil.py new file mode 100755 index 0000000..3b2d96c --- /dev/null +++ b/distro/src/bin/cputil.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys + +import atlas_config as mc + +DEFAULT_JVM_OPTS="-Xmx1024m" + +def main(): + + metadata_home = mc.metadataDir() + confdir = mc.dirMustExist(mc.confDir(metadata_home)) + logdir = mc.dirMustExist(mc.logDir(metadata_home)) + mc.executeEnvSh(confdir) + + jvm_opts_list = [] + + default_jvm_opts = DEFAULT_JVM_OPTS + metadata_jvm_opts = os.environ.get(mc.METADATA_OPTS, default_jvm_opts) + jvm_opts_list.extend(metadata_jvm_opts.split()) + + #expand web app dir + web_app_dir = mc.webAppDir(metadata_home) + mc.expandWebApp(metadata_home) + + p = os.pathsep + metadata_classpath = confdir + p \ + + os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \ + + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "*" ) + p \ + + os.path.join(metadata_home, "libext", "*") + + process = mc.java("org.apache.atlas.util.CredentialProviderUtility", sys.argv[1:], metadata_classpath, jvm_opts_list) + process.wait() + +if __name__ == '__main__': + try: + returncode = main() + except Exception as e: + print "Exception: %s " % str(e) + returncode = -1 + + sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/bin/quick_start.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/quick_start.py b/distro/src/bin/quick_start.py new file mode 100755 index 0000000..e30ba3d --- /dev/null +++ b/distro/src/bin/quick_start.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys + +import atlas_config as mc + +METADATA_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=quick_start.log" +METADATA_COMMAND_OPTS="-Dmetadata.home=%s" +DEFAULT_JVM_OPTS="-Xmx1024m -Dlog4j.configuration=atlas-log4j.xml" + +def main(): + + metadata_home = mc.metadataDir() + confdir = mc.dirMustExist(mc.confDir(metadata_home)) + mc.executeEnvSh(confdir) + logdir = mc.dirMustExist(mc.logDir(metadata_home)) + + #create sys property for conf dirs + jvm_opts_list = (METADATA_LOG_OPTS % logdir).split() + + cmd_opts = (METADATA_COMMAND_OPTS % metadata_home) + jvm_opts_list.extend(cmd_opts.split()) + + default_jvm_opts = DEFAULT_JVM_OPTS + metadata_jvm_opts = os.environ.get(mc.METADATA_OPTS, default_jvm_opts) + jvm_opts_list.extend(metadata_jvm_opts.split()) + + #expand web app dir + web_app_dir = mc.webAppDir(metadata_home) + mc.expandWebApp(metadata_home) + + p = os.pathsep + metadata_classpath = confdir + p \ + + os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \ + + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "*" ) + p \ + + os.path.join(metadata_home, "libext", "*") + + process = mc.java("org.apache.atlas.examples.QuickStart", sys.argv[1:], metadata_classpath, jvm_opts_list) + process.wait() + + print "Example data added to Apache Atlas Server!!!\n" + +if __name__ == '__main__': + try: + returncode = main() + except Exception as e: + print "Exception: %s " % str(e) + returncode = -1 + + sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/conf/application.properties ---------------------------------------------------------------------- diff --git a/distro/src/conf/application.properties b/distro/src/conf/application.properties new file mode 100755 index 0000000..bf323a7 --- /dev/null +++ b/distro/src/conf/application.properties @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +######### Graph Database Configs ######### +# Graph Storage +atlas.graph.storage.backend=berkeleyje +atlas.graph.storage.directory=${sys:atlas.home}/data/berkley + +#Hbase as stoarge backend +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +#atlas.graph.storage.hostname=localhost + +#Solr +#atlas.graph.index.search.backend=solr + +# Solr cloud mode properties +#atlas.graph.index.search.solr.mode=cloud +#atlas.graph.index.search.solr.zookeeper-url=localhost:2181 + +#Solr http mode properties +#atlas.graph.index.search.solr.mode=http +#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr + +# Graph Search Index +atlas.graph.index.search.backend=elasticsearch +atlas.graph.index.search.directory=${sys:atlas.home}/data/es +atlas.graph.index.search.elasticsearch.client-only=false +atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +######### Notification Configs ######### +atlas.notification.embedded=true +atlas.notification.kafka.data=${sys:atlas.home}/data/kafka + +######### Hive Lineage Configs ######### +# This models reflects the base super types for Data and Process +#atlas.lineage.hive.table.type.name=DataSet +#atlas.lineage.hive.process.type.name=Process +#atlas.lineage.hive.process.inputs.name=inputs +#atlas.lineage.hive.process.outputs.name=outputs + +## Schema +atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns +atlas.lineage.hive.table.schema.query.Table=Table where name='%s'\, columns + +## Server port configuration +#atlas.server.http.port=21000 +#atlas.server.https.port=21443 + +######### Security Properties ######### + +# SSL config +atlas.enableTLS=false + +######### Security Properties ######### http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/conf/atlas-env.sh ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-env.sh b/distro/src/conf/atlas-env.sh new file mode 100644 index 0000000..4b82e0c --- /dev/null +++ b/distro/src/conf/atlas-env.sh @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path +#export JAVA_HOME= + +# any additional java opts you want to set. This will apply to both client and server operations +#export METADATA_OPTS= + +# any additional java opts that you want to set for client only +#export METADATA_CLIENT_OPTS= + +# java heap size we want to set for the client. Default is 1024MB +#export METADATA_CLIENT_HEAP= + +# any additional opts you want to set for atlas service. +#export METADATA_SERVER_OPTS= + +# java heap size we want to set for the atlas server. Default is 1024MB +#export METADATA_SERVER_HEAP= + +# What is is considered as atlas home dir. Default is the base locaion of the installed software +#export METADATA_HOME_DIR= + +# Where log files are stored. Defatult is logs directory under the base install location +#export METADATA_LOG_DIR= + +# Where pid files are stored. Defatult is logs directory under the base install location +#export METADATA_PID_DIR= + +# where the atlas titan db data is stored. Defatult is logs/data directory under the base install location +#export METADATA_DATA_DIR= + +# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. +#export METADATA_EXPANDED_WEBAPP_DIR= http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/conf/atlas-log4j.xml ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-log4j.xml b/distro/src/conf/atlas-log4j.xml new file mode 100755 index 0000000..887cae5 --- /dev/null +++ b/distro/src/conf/atlas-log4j.xml @@ -0,0 +1,64 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/> + </layout> + </appender> + + <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="File" value="${atlas.log.dir}/${atlas.log.file}"/> + <param name="Append" value="true"/> + <param name="Threshold" value="debug"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/> + </layout> + </appender> + + <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="File" value="${atlas.log.dir}/audit.log"/> + <param name="Append" value="true"/> + <param name="Threshold" value="debug"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %x %m%n"/> + </layout> + </appender> + + <logger name="org.apache.atlas" additivity="false"> + <level value="debug"/> + <appender-ref ref="FILE"/> + </logger> + + + <logger name="AUDIT"> + <level value="info"/> + <appender-ref ref="AUDIT"/> + </logger> + + <root> + <priority value="info"/> + <appender-ref ref="FILE"/> + </root> + +</log4j:configuration> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/conf/client.properties ---------------------------------------------------------------------- diff --git a/distro/src/conf/client.properties b/distro/src/conf/client.properties new file mode 100755 index 0000000..ab6ef2f --- /dev/null +++ b/distro/src/conf/client.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +######### Security Properties ######### + +# SSL config + +atlas.enableTLS=false + +#truststore.file=/path/to/truststore.jks +#cert.stores.credential.provider.path=jceks://file/path/to/credentialstore.jceks + +#following only required for 2-way SSL +#keystore.file=/path/to/keystore.jks + +# Authentication config + +# enabled: true or false +atlas.http.authentication.enabled=false +# type: simple or kerberos +atlas.http.authentication.type=simple + +######### Security Properties ######### http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/conf/solr/currency.xml ---------------------------------------------------------------------- diff --git a/distro/src/conf/solr/currency.xml b/distro/src/conf/solr/currency.xml new file mode 100644 index 0000000..3a9c58a --- /dev/null +++ b/distro/src/conf/solr/currency.xml @@ -0,0 +1,67 @@ +<?xml version="1.0" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- Example exchange rates file for CurrencyField type named "currency" in example schema --> + +<currencyConfig version="1.0"> + <rates> + <!-- Updated from http://www.exchangerate.com/ at 2011-09-27 --> + <rate from="USD" to="ARS" rate="4.333871" comment="ARGENTINA Peso" /> + <rate from="USD" to="AUD" rate="1.025768" comment="AUSTRALIA Dollar" /> + <rate from="USD" to="EUR" rate="0.743676" comment="European Euro" /> + <rate from="USD" to="BRL" rate="1.881093" comment="BRAZIL Real" /> + <rate from="USD" to="CAD" rate="1.030815" comment="CANADA Dollar" /> + <rate from="USD" to="CLP" rate="519.0996" comment="CHILE Peso" /> + <rate from="USD" to="CNY" rate="6.387310" comment="CHINA Yuan" /> + <rate from="USD" to="CZK" rate="18.47134" comment="CZECH REP. Koruna" /> + <rate from="USD" to="DKK" rate="5.515436" comment="DENMARK Krone" /> + <rate from="USD" to="HKD" rate="7.801922" comment="HONG KONG Dollar" /> + <rate from="USD" to="HUF" rate="215.6169" comment="HUNGARY Forint" /> + <rate from="USD" to="ISK" rate="118.1280" comment="ICELAND Krona" /> + <rate from="USD" to="INR" rate="49.49088" comment="INDIA Rupee" /> + <rate from="USD" to="XDR" rate="0.641358" comment="INTNL MON. FUND SDR" /> + <rate from="USD" to="ILS" rate="3.709739" comment="ISRAEL Sheqel" /> + <rate from="USD" to="JPY" rate="76.32419" comment="JAPAN Yen" /> + <rate from="USD" to="KRW" rate="1169.173" comment="KOREA (SOUTH) Won" /> + <rate from="USD" to="KWD" rate="0.275142" comment="KUWAIT Dinar" /> + <rate from="USD" to="MXN" rate="13.85895" comment="MEXICO Peso" /> + <rate from="USD" to="NZD" rate="1.285159" comment="NEW ZEALAND Dollar" /> + <rate from="USD" to="NOK" rate="5.859035" comment="NORWAY Krone" /> + <rate from="USD" to="PKR" rate="87.57007" comment="PAKISTAN Rupee" /> + <rate from="USD" to="PEN" rate="2.730683" comment="PERU Sol" /> + <rate from="USD" to="PHP" rate="43.62039" comment="PHILIPPINES Peso" /> + <rate from="USD" to="PLN" rate="3.310139" comment="POLAND Zloty" /> + <rate from="USD" to="RON" rate="3.100932" comment="ROMANIA Leu" /> + <rate from="USD" to="RUB" rate="32.14663" comment="RUSSIA Ruble" /> + <rate from="USD" to="SAR" rate="3.750465" comment="SAUDI ARABIA Riyal" /> + <rate from="USD" to="SGD" rate="1.299352" comment="SINGAPORE Dollar" /> + <rate from="USD" to="ZAR" rate="8.329761" comment="SOUTH AFRICA Rand" /> + <rate from="USD" to="SEK" rate="6.883442" comment="SWEDEN Krona" /> + <rate from="USD" to="CHF" rate="0.906035" comment="SWITZERLAND Franc" /> + <rate from="USD" to="TWD" rate="30.40283" comment="TAIWAN Dollar" /> + <rate from="USD" to="THB" rate="30.89487" comment="THAILAND Baht" /> + <rate from="USD" to="AED" rate="3.672955" comment="U.A.E. Dirham" /> + <rate from="USD" to="UAH" rate="7.988582" comment="UKRAINE Hryvnia" /> + <rate from="USD" to="GBP" rate="0.647910" comment="UNITED KINGDOM Pound" /> + + <!-- Cross-rates for some common currencies --> + <rate from="EUR" to="GBP" rate="0.869914" /> + <rate from="EUR" to="NOK" rate="7.800095" /> + <rate from="GBP" to="NOK" rate="8.966508" /> + </rates> +</currencyConfig> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/conf/solr/lang/stopwords_en.txt ---------------------------------------------------------------------- diff --git a/distro/src/conf/solr/lang/stopwords_en.txt b/distro/src/conf/solr/lang/stopwords_en.txt new file mode 100644 index 0000000..2c164c0 --- /dev/null +++ b/distro/src/conf/solr/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/conf/solr/protwords.txt ---------------------------------------------------------------------- diff --git a/distro/src/conf/solr/protwords.txt b/distro/src/conf/solr/protwords.txt new file mode 100644 index 0000000..1dfc0ab --- /dev/null +++ b/distro/src/conf/solr/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e77d3d33/distro/src/conf/solr/schema.xml ---------------------------------------------------------------------- diff --git a/distro/src/conf/solr/schema.xml b/distro/src/conf/solr/schema.xml new file mode 100644 index 0000000..1d445b1 --- /dev/null +++ b/distro/src/conf/solr/schema.xml @@ -0,0 +1,534 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + This is the Solr schema file. This file should be named "schema.xml" and + should be in the conf directory under the solr home + (i.e. ./solr/conf/schema.xml by default) + or located where the classloader for the Solr webapp can find it. + + This example schema is the recommended starting point for users. + It should be kept correct and concise, usable out-of-the-box. + + For more information, on how to customize this file, please see + http://wiki.apache.org/solr/SchemaXml +--> + +<schema name="titan-schema" version="1.5"> + <!-- attribute "name" is the name of this schema and is only used for display purposes. + version="x.y" is Solr's version number for the schema syntax and + semantics. It should not normally be changed by applications. + + 1.0: multiValued attribute did not exist, all fields are multiValued + by nature + 1.1: multiValued attribute introduced, false by default + 1.2: omitTermFreqAndPositions attribute introduced, true by default + except for text fields. + 1.3: removed optional field compress feature + 1.4: autoGeneratePhraseQueries attribute introduced to drive QueryParser + behavior when a single string produces multiple tokens. Defaults + to off for version >= 1.4 + 1.5: omitNorms defaults to true for primitive field types + (int, float, boolean, string...) + --> + + + <!-- Valid attributes for fields: + name: mandatory - the name for the field + type: mandatory - the name of a field type from the + <types> fieldType section + indexed: true if this field should be indexed (searchable or sortable) + stored: true if this field should be retrievable + docValues: true if this field should have doc values. Doc values are + useful for faceting, grouping, sorting and function queries. Although not + required, doc values will make the index faster to load, more + NRT-friendly and more memory-efficient. They however come with some + limitations: they are currently only supported by StrField, UUIDField + and all Trie*Fields, and depending on the field type, they might + require the field to be single-valued, be required or have a default + value (check the documentation of the field type you're interested in + for more information) + multiValued: true if this field may contain multiple values per document + omitNorms: (expert) set to true to omit the norms associated with + this field (this disables length normalization and index-time + boosting for the field, and saves some memory). Only full-text + fields or fields that need an index-time boost need norms. + Norms are omitted for primitive (non-analyzed) types by default. + termVectors: [false] set to true to store the term vector for a + given field. + When using MoreLikeThis, fields used for similarity should be + stored for best performance. + termPositions: Store position information with the term vector. + This will increase storage costs. + termOffsets: Store offset information with the term vector. This + will increase storage costs. + required: The field is required. It will throw an error if the + value does not exist + default: a value that should be used if no value is specified + when adding a document. + --> + + <!-- field names should consist of alphanumeric or underscore characters only and + not start with a digit. This is not currently strictly enforced, + but other field names will not have first class support from all components + and back compatibility is not guaranteed. Names with both leading and + trailing underscores (e.g. _version_) are reserved. + --> + + <!-- If you remove this field, you must _also_ disable the update log in solrconfig.xml + or Solr won't start. _version_ and update log are required for SolrCloud + --> + <field name="_version_" type="long" indexed="true" stored="true"/> + + <!-- points to the root document of a block of nested documents. Required for nested + document support, may be removed otherwise + --> + <field name="_root_" type="string" indexed="true" stored="false"/> + + <!-- Only remove the "id" field if you have a very good reason to. While not strictly + required, it is highly recommended. A <uniqueKey> is present in almost all Solr + installations. See the <uniqueKey> declaration below where <uniqueKey> is set to "id". + --> + <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" /> + + <!-- Dynamic field definitions allow using convention over configuration + for fields via the specification of patterns to match field names. + EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i) + RESTRICTION: the glob-like pattern in the name attribute must have + a "*" only at the start or the end. --> + + <dynamicField name="*_i" type="int" indexed="true" stored="true"/> + <dynamicField name="*_is" type="int" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_s" type="string" indexed="true" stored="true" /> + <dynamicField name="*_ss" type="string" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_l" type="long" indexed="true" stored="true"/> + <dynamicField name="*_ls" type="long" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_t" type="text_general" indexed="true" stored="true"/> + <dynamicField name="*_txt" type="text_general" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_en" type="text_en" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/> + <dynamicField name="*_bs" type="boolean" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_f" type="float" indexed="true" stored="true"/> + <dynamicField name="*_fs" type="float" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_d" type="double" indexed="true" stored="true"/> + <dynamicField name="*_ds" type="double" indexed="true" stored="true" multiValued="true"/> + + <!-- Type used to index the lat and lon components for the "location" FieldType --> + <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" /> + + <dynamicField name="*_dt" type="date" indexed="true" stored="true"/> + <dynamicField name="*_dts" type="date" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_p" type="location" indexed="true" stored="true"/> + + <!-- some trie-coded dynamic fields for faster range queries --> + <dynamicField name="*_ti" type="tint" indexed="true" stored="true"/> + <dynamicField name="*_tl" type="tlong" indexed="true" stored="true"/> + <dynamicField name="*_tf" type="tfloat" indexed="true" stored="true"/> + <dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/> + <dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/> + + <dynamicField name="*_c" type="currency" indexed="true" stored="true"/> + + <dynamicField name="ignored_*" type="ignored" multiValued="true"/> + <dynamicField name="attr_*" type="text_general" indexed="true" stored="true" multiValued="true"/> + + <dynamicField name="random_*" type="random" /> + + <!-- uncomment the following to ignore any fields that don't already match an existing + field name or dynamic field, rather than reporting them as an error. + alternately, change the type="ignored" to some other type e.g. "text" if you want + unknown fields indexed and/or stored by default --> + <!--dynamicField name="*" type="ignored" multiValued="true" /--> + + <!-- Field to use to determine and enforce document uniqueness. + Unless this field is marked with required="false", it will be a required field + --> + <uniqueKey>id</uniqueKey> + + <!-- copyField commands copy one field to another at the time a document + is added to the index. It's used either to index the same field differently, + or to add multiple fields to the same field for easier/faster searching. --> + + <!-- + <copyField source="title" dest="text"/> + <copyField source="body" dest="text"/> + --> + + <!-- field type definitions. The "name" attribute is + just a label to be used by field definitions. The "class" + attribute and any other attributes determine the real + behavior of the fieldType. + Class names starting with "solr" refer to java classes in a + standard package such as org.apache.solr.analysis + --> + + <!-- The StrField type is not analyzed, but indexed/stored verbatim. + It supports doc values but in that case the field needs to be + single-valued and either required or have a default value. + --> + <fieldType name="string" class="solr.StrField" sortMissingLast="true" /> + + <!-- boolean type: "true" or "false" --> + <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/> + + <!-- sortMissingLast and sortMissingFirst attributes are optional attributes are + currently supported on types that are sorted internally as strings + and on numeric types. + This includes "string","boolean", and, as of 3.5 (and 4.x), + int, float, long, date, double, including the "Trie" variants. + - If sortMissingLast="true", then a sort on this field will cause documents + without the field to come after documents with the field, + regardless of the requested sort order (asc or desc). + - If sortMissingFirst="true", then a sort on this field will cause documents + without the field to come before documents with the field, + regardless of the requested sort order. + - If sortMissingLast="false" and sortMissingFirst="false" (the default), + then default lucene sorting will be used which places docs without the + field first in an ascending sort and last in a descending sort. + --> + + <!-- + Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types. + + These fields support doc values, but they require the field to be + single-valued and either be required or have a default value. + --> + <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/> + + <!-- + Numeric field types that index each value at various levels of precision + to accelerate range queries when the number of values between the range + endpoints is large. See the javadoc for NumericRangeQuery for internal + implementation details. + + Smaller precisionStep values (specified in bits) will lead to more tokens + indexed per value, slightly larger index size, and faster range queries. + A precisionStep of 0 disables indexing at different precision levels. + --> + <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/> + <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/> + <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/> + <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/> + + <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and + is a more restricted form of the canonical representation of dateTime + http://www.w3.org/TR/xmlschema-2/#dateTime + The trailing "Z" designates UTC time and is mandatory. + Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z + All other components are mandatory. + + Expressions can also be used to denote calculations that should be + performed relative to "NOW" to determine the value, ie... + + NOW/HOUR + ... Round to the start of the current hour + NOW-1DAY + ... Exactly 1 day prior to now + NOW/DAY+6MONTHS+3DAYS + ... 6 months and 3 days in the future from the start of + the current day + + Consult the TrieDateField javadocs for more information. + + Note: For faster range queries, consider the tdate type + --> + <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/> + + <!-- A Trie based date field for faster date range queries and date faceting. --> + <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/> + + + <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings --> + <fieldType name="binary" class="solr.BinaryField"/> + + <!-- The "RandomSortField" is not used to store or search any + data. You can declare fields of this type it in your schema + to generate pseudo-random orderings of your docs for sorting + or function purposes. The ordering is generated based on the field + name and the version of the index. As long as the index version + remains unchanged, and the same field name is reused, + the ordering of the docs will be consistent. + If you want different psuedo-random orderings of documents, + for the same version of the index, use a dynamicField and + change the field name in the request. + --> + <fieldType name="random" class="solr.RandomSortField" indexed="true" /> + + <!-- solr.TextField allows the specification of custom text analyzers + specified as a tokenizer and a list of token filters. Different + analyzers may be specified for indexing and querying. + + The optional positionIncrementGap puts space between multiple fields of + this type on the same document, with the purpose of preventing false phrase + matching across fields. + + For more info on customizing your analyzer chain, please see + http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters + --> + + <!-- One can also specify an existing Analyzer class that has a + default constructor via the class attribute on the analyzer element. + Example: + <fieldType name="text_greek" class="solr.TextField"> + <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/> + </fieldType> + --> + + <!-- A text field that only splits on whitespace for exact matching of words --> + <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + </analyzer> + </fieldType> + + <!-- A general text field that has reasonable, generic + cross-language defaults: it tokenizes with StandardTokenizer, + removes stop words from case-insensitive "stopwords.txt" + (empty by default), and down cases. At query time only, it + also applies synonyms. --> + <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100"> + <analyzer type="index"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" /> + <!-- in this example, we will only use synonyms at query time + <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/> + --> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" /> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + + <!-- A text field with defaults appropriate for English: it + tokenizes with StandardTokenizer, removes English stop words + (lang/stopwords_en.txt), down cases, protects words from protwords.txt, and + finally applies Porter's stemming. The query time analyzer + also applies synonyms from synonyms.txt. --> + <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100"> + <analyzer type="index"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <!-- in this example, we will only use synonyms at query time + <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/> + --> + <!-- Case insensitive stop word removal. + --> + <filter class="solr.StopFilterFactory" + ignoreCase="true" + words="lang/stopwords_en.txt" + /> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPossessiveFilterFactory"/> + <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/> + <!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory: + <filter class="solr.EnglishMinimalStemFilterFactory"/> + --> + <filter class="solr.PorterStemFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.StopFilterFactory" + ignoreCase="true" + words="lang/stopwords_en.txt" + /> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPossessiveFilterFactory"/> + <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/> + <!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory: + <filter class="solr.EnglishMinimalStemFilterFactory"/> + --> + <filter class="solr.PorterStemFilterFactory"/> + </analyzer> + </fieldType> + + <!-- A text field with defaults appropriate for English, plus + aggressive word-splitting and autophrase features enabled. + This field is just like text_en, except it adds + WordDelimiterFilter to enable splitting and matching of + words on case-change, alpha numeric boundaries, and + non-alphanumeric chars. This means certain compound word + cases will work, for example query "wi fi" will match + document "WiFi" or "wi-fi". + --> + <fieldType name="text_en_splitting" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true"> + <analyzer type="index"> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <!-- in this example, we will only use synonyms at query time + <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/> + --> + <!-- Case insensitive stop word removal. + --> + <filter class="solr.StopFilterFactory" + ignoreCase="true" + words="lang/stopwords_en.txt" + /> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/> + <filter class="solr.PorterStemFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.StopFilterFactory" + ignoreCase="true" + words="lang/stopwords_en.txt" + /> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/> + <filter class="solr.PorterStemFilterFactory"/> + </analyzer> + </fieldType> + + <!-- Less flexible matching, but less false matches. Probably not ideal for product names, + but may be good for SKUs. Can insert dashes in the wrong place and still match. --> + <fieldType name="text_en_splitting_tight" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true"> + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_en.txt"/> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/> + <filter class="solr.EnglishMinimalStemFilterFactory"/> + <!-- this filter can remove any duplicate tokens that appear at the same position - sometimes + possible with WordDelimiterFilter in conjuncton with stemming. --> + <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> + </analyzer> + </fieldType> + + <!-- Just like text_general except it reverses the characters of + each token, to enable more efficient leading wildcard queries. --> + <fieldType name="text_general_rev" class="solr.TextField" positionIncrementGap="100"> + <analyzer type="index"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" /> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.ReversedWildcardFilterFactory" withOriginal="true" + maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" /> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + + <!-- This is an example of using the KeywordTokenizer along + With various TokenFilterFactories to produce a sortable field + that does not include some properties of the source text + --> + <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true"> + <analyzer> + <!-- KeywordTokenizer does no actual tokenizing, so the entire + input string is preserved as a single token + --> + <tokenizer class="solr.KeywordTokenizerFactory"/> + <!-- The LowerCase TokenFilter does what you expect, which can be + when you want your sorting to be case insensitive + --> + <filter class="solr.LowerCaseFilterFactory" /> + <!-- The TrimFilter removes any leading or trailing whitespace --> + <filter class="solr.TrimFilterFactory" /> + <!-- The PatternReplaceFilter gives you the flexibility to use + Java Regular expression to replace any sequence of characters + matching a pattern with an arbitrary replacement string, + which may include back references to portions of the original + string matched by the pattern. + + See the Java Regular Expression documentation for more + information on pattern and replacement string syntax. + + http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html + --> + <filter class="solr.PatternReplaceFilterFactory" + pattern="([^a-z])" replacement="" replace="all" + /> + </analyzer> + </fieldType> + + <!-- lowercases the entire field value, keeping it as a single token. --> + <fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.KeywordTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory" /> + </analyzer> + </fieldType> + + <!-- since fields of this type are by default not stored or indexed, + any data added to them will be ignored outright. --> + <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" /> + + <!-- This point type indexes the coordinates as separate fields (subFields) + If subFieldType is defined, it references a type, and a dynamic field + definition is created matching *___<typename>. Alternately, if + subFieldSuffix is defined, that is used to create the subFields. + Example: if subFieldType="double", then the coordinates would be + indexed in fields myloc_0___double,myloc_1___double. + Example: if subFieldSuffix="_d" then the coordinates would be indexed + in fields myloc_0_d,myloc_1_d + The subFields are an implementation detail of the fieldType, and end + users normally should not need to know about them. + --> + <fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/> + + <!-- A specialized field for geospatial search. If indexed, this fieldType must not be multivalued. --> + <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/> + + <!-- Spatial rectangle (bounding box) field. It supports most spatial predicates, and has + special relevancy modes: score=overlapRatio|area|area2D (local-param to the query). DocValues is recommended for + relevancy. --> + <fieldType name="bbox" class="solr.BBoxField" + geo="true" distanceUnits="kilometers" numberType="_bbox_coord" /> + <fieldType name="_bbox_coord" class="solr.TrieDoubleField" precisionStep="8" docValues="true" stored="false"/> + + <!-- Money/currency field type. See http://wiki.apache.org/solr/MoneyFieldType + Parameters: + defaultCurrency: Specifies the default currency if none specified. Defaults to "USD" + precisionStep: Specifies the precisionStep for the TrieLong field used for the amount + providerClass: Lets you plug in other exchange provider backend: + solr.FileExchangeRateProvider is the default and takes one parameter: + currencyConfig: name of an xml file holding exchange rates + solr.OpenExchangeRatesOrgProvider uses rates from openexchangerates.org: + ratesFileLocation: URL or path to rates JSON file (default latest.json on the web) + refreshInterval: Number of minutes between each rates fetch (default: 1440, min: 60) + --> + <fieldType name="currency" class="solr.CurrencyField" precisionStep="8" defaultCurrency="USD" currencyConfig="currency.xml" /> + + <!--Titan specific--> + <fieldType name="uuid" + class="solr.UUIDField" + indexed="true" /> + + <dynamicField name="*_uuid" type="uuid" indexed="true" stored="true"/> + + <!-- TTL --> + <field name="ttl" type="string" indexed="true" stored="true" /> + <field name="expire_at" type="date" indexed="true" stored="true" /> + <field name="timestamp" type="date" indexed="true" stored="true" /> + +</schema>
