http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/atlas_config.py ---------------------------------------------------------------------- diff --git a/src/bin/atlas_config.py b/src/bin/atlas_config.py new file mode 100755 index 0000000..f8ffe45 --- /dev/null +++ b/src/bin/atlas_config.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import getpass + +import os +import platform +import subprocess +from threading import Thread +import sys +import time +import errno + +LIB = "lib" +CONF = "conf" +LOG="logs" +WEBAPP="server" + os.sep + "webapp" +DATA="data" +ENV_KEYS = ["JAVA_HOME", "METADATA_OPTS", "METADATA_LOG_DIR", "METADATA_PID_DIR", "METADATA_CONF", "METADATACPPATH", "METADATA_DATA_DIR", "METADATA_HOME_DIR", "METADATA_EXPANDED_WEBAPP_DIR"] +METADATA_CONF = "METADATA_CONF" +METADATA_LOG = "METADATA_LOG_DIR" +METADATA_PID = "METADATA_PID_DIR" +METADATA_WEBAPP = "METADATA_EXPANDED_WEBAPP_DIR" +METADATA_OPTS = "METADATA_OPTS" +METADATA_DATA = "METADATA_DATA_DIR" +METADATA_HOME = "METADATA_HOME_DIR" +IS_WINDOWS = platform.system() == "Windows" +ON_POSIX = 'posix' in sys.builtin_module_names +DEBUG = False + +def scriptDir(): + """ + get the script path + """ + return os.path.dirname(os.path.realpath(__file__)) + +def metadataDir(): + home = os.path.dirname(scriptDir()) + return os.environ.get(METADATA_HOME, home) + +def libDir(dir) : + return os.path.join(dir, LIB) + +def confDir(dir): + localconf = os.path.join(dir, CONF) + return os.environ.get(METADATA_CONF, localconf) + +def logDir(dir): + localLog = os.path.join(dir, LOG) + return os.environ.get(METADATA_LOG, localLog) + +def pidFile(dir): + localPid = os.path.join(dir, LOG) + return os.path.join(os.environ.get(METADATA_PID, localPid), 'metadata.pid') + +def dataDir(dir): + data = os.path.join(dir, DATA) + return os.environ.get(METADATA_DATA, data) + +def webAppDir(dir): + webapp = os.path.join(dir, WEBAPP) + return os.environ.get(METADATA_WEBAPP, webapp) + +def expandWebApp(dir): + webappDir = webAppDir(dir) + webAppMetadataDir = os.path.join(webappDir, "metadata") + d = os.sep + if not os.path.exists(os.path.join(webAppMetadataDir, "WEB-INF")): + try: + os.makedirs(webAppMetadataDir) + except OSError, e: + if e.errno != errno.EEXIST: + raise e + pass + os.chdir(webAppMetadataDir) + jar(os.path.join(metadataDir(), "server", "webapp", "atlas.war")) + +def dirMustExist(dirname): + if not os.path.exists(dirname): + os.mkdir(dirname) + return dirname + +def executeEnvSh(confDir): + envscript = '%s/metadata-env.sh' % confDir + if not IS_WINDOWS and os.path.exists(envscript): + envCmd = 'source %s && env' % envscript + command = ['bash', '-c', envCmd] + + proc = subprocess.Popen(command, stdout = subprocess.PIPE) + + for line in proc.stdout: + (key, _, value) = line.strip().partition("=") + if key in ENV_KEYS: + os.environ[key] = value + + proc.communicate() + +def java(classname, args, classpath, jvm_opts_list, logdir=None): + java_home = os.environ.get("JAVA_HOME", None) + if java_home: + prg = os.path.join(java_home, "bin", "java") + else: + prg = which("java") + + commandline = [prg] + commandline.extend(jvm_opts_list) + commandline.append("-classpath") + commandline.append(classpath) + commandline.append(classname) + commandline.extend(args) + return runProcess(commandline, logdir) + +def jar(path): + java_home = os.environ.get("JAVA_HOME", None) + if java_home: + prg = os.path.join(java_home, "bin", "jar") + else: + prg = which("jar") + + commandline = [prg] + commandline.append("-xf") + commandline.append(path) + process = runProcess(commandline) + process.wait() + +def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + +def which(program): + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + return None + +def runProcess(commandline, logdir=None): + """ + Run a process + :param commandline: command line + :return:the return code + """ + global finished + debug ("Executing : %s" % commandline) + timestr = time.strftime("metadata.%Y%m%d-%H%M%S") + stdoutFile = None + stderrFile = None + if logdir: + stdoutFile = open(os.path.join(logdir, timestr + ".out"), "w") + stderrFile = open(os.path.join(logdir,timestr + ".err"), "w") + return subprocess.Popen(commandline, stdout=stdoutFile, stderr=stderrFile) + +def print_output(name, src, toStdErr): + """ + Relay the output stream to stdout line by line + :param name: + :param src: source stream + :param toStdErr: flag set if stderr is to be the dest + :return: + """ + + global needPassword + debug ("starting printer for %s" % name ) + line = "" + while not finished: + (line, done) = read(src, line) + if done: + out(toStdErr, line + "\n") + flush(toStdErr) + if line.find("Enter password for") >= 0: + needPassword = True + line = "" + out(toStdErr, line) + # closedown: read remainder of stream + c = src.read(1) + while c!="" : + c = c.decode('utf-8') + out(toStdErr, c) + if c == "\n": + flush(toStdErr) + c = src.read(1) + flush(toStdErr) + src.close() + +def read_input(name, exe): + """ + Read input from stdin and send to process + :param name: + :param process: process to send input to + :return: + """ + global needPassword + debug ("starting reader for %s" % name ) + while not finished: + if needPassword: + needPassword = False + if sys.stdin.isatty(): + cred = getpass.getpass() + else: + cred = sys.stdin.readline().rstrip() + exe.stdin.write(cred + "\n") + +def debug(text): + if DEBUG: print '[DEBUG] ' + text + + +def error(text): + print '[ERROR] ' + text + sys.stdout.flush() + +def info(text): + print text + sys.stdout.flush() + + +def out(toStdErr, text) : + """ + Write to one of the system output channels. + This action does not add newlines. If you want that: write them yourself + :param toStdErr: flag set if stderr is to be the dest + :param text: text to write. + :return: + """ + if toStdErr: + sys.stderr.write(text) + else: + sys.stdout.write(text) + +def flush(toStdErr) : + """ + Flush the output stream + :param toStdErr: flag set if stderr is to be the dest + :return: + """ + if toStdErr: + sys.stderr.flush() + else: + sys.stdout.flush() + +def read(pipe, line): + """ + read a char, append to the listing if there is a char that is not \n + :param pipe: pipe to read from + :param line: line being built up + :return: (the potentially updated line, flag indicating newline reached) + """ + + c = pipe.read(1) + if c != "": + o = c.decode('utf-8') + if o != '\n': + line += o + return line, False + else: + return line, True + else: + return line, False + +def writePid(metadata_pid_file, process): + f = open(metadata_pid_file, 'w') + f.write(str(process.pid)) + f.close() + +
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/atlas_start.py ---------------------------------------------------------------------- diff --git a/src/bin/atlas_start.py b/src/bin/atlas_start.py new file mode 100755 index 0000000..74b242b --- /dev/null +++ b/src/bin/atlas_start.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +import traceback + +import atlas_config as mc + +METADATA_LOG_OPTS="-Dmetadata.log.dir=%s -Dmetadata.log.file=application.log" +METADATA_COMMAND_OPTS="-Dmetadata.home=%s" +METADATA_CONFIG_OPTS="-Dmetadata.conf=%s" +DEFAULT_JVM_OPTS="-Xmx1024m" + +def main(): + + metadata_home = mc.metadataDir() + confdir = mc.dirMustExist(mc.confDir(metadata_home)) + mc.executeEnvSh(confdir) + logdir = mc.dirMustExist(mc.logDir(metadata_home)) + + #create sys property for conf dirs + jvm_opts_list = (METADATA_LOG_OPTS % logdir).split() + + cmd_opts = (METADATA_COMMAND_OPTS % metadata_home) + jvm_opts_list.extend(cmd_opts.split()) + + config_opts = (METADATA_CONFIG_OPTS % confdir) + jvm_opts_list.extend(config_opts.split()) + + default_jvm_opts = DEFAULT_JVM_OPTS + metadata_jvm_opts = os.environ.get(mc.METADATA_OPTS, default_jvm_opts) + jvm_opts_list.extend(metadata_jvm_opts.split()) + + #expand web app dir + web_app_dir = mc.webAppDir(metadata_home) + mc.expandWebApp(metadata_home) + + p = os.pathsep + metadata_classpath = confdir + p \ + + os.path.join(web_app_dir, "metadata", "WEB-INF", "classes" ) + p \ + + os.path.join(web_app_dir, "metadata", "WEB-INF", "lib", "*" ) + p \ + + os.path.join(metadata_home, "libext", "*") + + metadata_pid_file = mc.pidFile(metadata_home) + + if os.path.isfile(metadata_pid_file): + print "%s already exists, exiting" % metadata_pid_file + sys.exit() + + args = ["-app", os.path.join(web_app_dir, "metadata")] + args.extend(sys.argv[1:]) + + process = mc.java("org.apache.atlas.Main", args, metadata_classpath, jvm_opts_list, logdir) + mc.writePid(metadata_pid_file, process) + + print "Apache Atlas Server started!!!\n" + +if __name__ == '__main__': + try: + returncode = main() + except Exception as e: + print "Exception: %s " % str(e) + print traceback.format_exc() + returncode = -1 + + sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/atlas_stop.py ---------------------------------------------------------------------- diff --git a/src/bin/atlas_stop.py b/src/bin/atlas_stop.py new file mode 100755 index 0000000..a157427 --- /dev/null +++ b/src/bin/atlas_stop.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from signal import SIGTERM +import sys +import traceback + +import atlas_config as mc + +def main(): + + metadata_home = mc.metadataDir() + confdir = mc.dirMustExist(mc.confDir(metadata_home)) + mc.executeEnvSh(confdir) + piddir = mc.dirMustExist(mc.logDir(metadata_home)) + + metadata_pid_file = mc.pidFile(metadata_home) + + try: + pf = file(metadata_pid_file, 'r') + pid = int(pf.read().strip()) + pf.close() + except: + pid = None + + if not pid: + sys.stderr.write("No process ID file found. Server not running?\n") + return + + os.kill(pid, SIGTERM) + + # assuming kill worked since process check on windows is more involved... + if os.path.exists(metadata_pid_file): + os.remove(metadata_pid_file) + +if __name__ == '__main__': + try: + returncode = main() + except Exception as e: + print "Exception: %s " % str(e) + print traceback.format_exc() + returncode = -1 + + sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/metadata_config.py ---------------------------------------------------------------------- diff --git a/src/bin/metadata_config.py b/src/bin/metadata_config.py deleted file mode 100755 index 8c688ab..0000000 --- a/src/bin/metadata_config.py +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env python - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import getpass - -import os -import platform -import subprocess -from threading import Thread -import sys -import time -import errno - -LIB = "lib" -CONF = "conf" -LOG="logs" -WEBAPP="server" + os.sep + "webapp" -DATA="data" -ENV_KEYS = ["JAVA_HOME", "METADATA_OPTS", "METADATA_LOG_DIR", "METADATA_PID_DIR", "METADATA_CONF", "METADATACPPATH", "METADATA_DATA_DIR", "METADATA_HOME_DIR", "METADATA_EXPANDED_WEBAPP_DIR"] -METADATA_CONF = "METADATA_CONF" -METADATA_LOG = "METADATA_LOG_DIR" -METADATA_PID = "METADATA_PID_DIR" -METADATA_WEBAPP = "METADATA_EXPANDED_WEBAPP_DIR" -METADATA_OPTS = "METADATA_OPTS" -METADATA_DATA = "METADATA_DATA_DIR" -METADATA_HOME = "METADATA_HOME_DIR" -IS_WINDOWS = platform.system() == "Windows" -ON_POSIX = 'posix' in sys.builtin_module_names -DEBUG = False - -def scriptDir(): - """ - get the script path - """ - return os.path.dirname(os.path.realpath(__file__)) - -def metadataDir(): - home = os.path.dirname(scriptDir()) - return os.environ.get(METADATA_HOME, home) - -def libDir(dir) : - return os.path.join(dir, LIB) - -def confDir(dir): - localconf = os.path.join(dir, CONF) - return os.environ.get(METADATA_CONF, localconf) - -def logDir(dir): - localLog = os.path.join(dir, LOG) - return os.environ.get(METADATA_LOG, localLog) - -def pidFile(dir): - localPid = os.path.join(dir, LOG) - return os.path.join(os.environ.get(METADATA_PID, localPid), 'metadata.pid') - -def dataDir(dir): - data = os.path.join(dir, DATA) - return os.environ.get(METADATA_DATA, data) - -def webAppDir(dir): - webapp = os.path.join(dir, WEBAPP) - return os.environ.get(METADATA_WEBAPP, webapp) - -def expandWebApp(dir): - webappDir = webAppDir(dir) - webAppMetadataDir = os.path.join(webappDir, "metadata") - d = os.sep - if not os.path.exists(os.path.join(webAppMetadataDir, "WEB-INF")): - try: - os.makedirs(webAppMetadataDir) - except OSError, e: - if e.errno != errno.EEXIST: - raise e - pass - os.chdir(webAppMetadataDir) - jar(os.path.join(metadataDir(), "server", "webapp", "metadata.war")) - -def dirMustExist(dirname): - if not os.path.exists(dirname): - os.mkdir(dirname) - return dirname - -def executeEnvSh(confDir): - envscript = '%s/metadata-env.sh' % confDir - if not IS_WINDOWS and os.path.exists(envscript): - envCmd = 'source %s && env' % envscript - command = ['bash', '-c', envCmd] - - proc = subprocess.Popen(command, stdout = subprocess.PIPE) - - for line in proc.stdout: - (key, _, value) = line.strip().partition("=") - if key in ENV_KEYS: - os.environ[key] = value - - proc.communicate() - -def java(classname, args, classpath, jvm_opts_list, logdir=None): - java_home = os.environ.get("JAVA_HOME", None) - if java_home: - prg = os.path.join(java_home, "bin", "java") - else: - prg = which("java") - - commandline = [prg] - commandline.extend(jvm_opts_list) - commandline.append("-classpath") - commandline.append(classpath) - commandline.append(classname) - commandline.extend(args) - return runProcess(commandline, logdir) - -def jar(path): - java_home = os.environ.get("JAVA_HOME", None) - if java_home: - prg = os.path.join(java_home, "bin", "jar") - else: - prg = which("jar") - - commandline = [prg] - commandline.append("-xf") - commandline.append(path) - process = runProcess(commandline) - process.wait() - -def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - -def which(program): - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - return None - -def runProcess(commandline, logdir=None): - """ - Run a process - :param commandline: command line - :return:the return code - """ - global finished - debug ("Executing : %s" % commandline) - timestr = time.strftime("metadata.%Y%m%d-%H%M%S") - stdoutFile = None - stderrFile = None - if logdir: - stdoutFile = open(os.path.join(logdir, timestr + ".out"), "w") - stderrFile = open(os.path.join(logdir,timestr + ".err"), "w") - return subprocess.Popen(commandline, stdout=stdoutFile, stderr=stderrFile) - -def print_output(name, src, toStdErr): - """ - Relay the output stream to stdout line by line - :param name: - :param src: source stream - :param toStdErr: flag set if stderr is to be the dest - :return: - """ - - global needPassword - debug ("starting printer for %s" % name ) - line = "" - while not finished: - (line, done) = read(src, line) - if done: - out(toStdErr, line + "\n") - flush(toStdErr) - if line.find("Enter password for") >= 0: - needPassword = True - line = "" - out(toStdErr, line) - # closedown: read remainder of stream - c = src.read(1) - while c!="" : - c = c.decode('utf-8') - out(toStdErr, c) - if c == "\n": - flush(toStdErr) - c = src.read(1) - flush(toStdErr) - src.close() - -def read_input(name, exe): - """ - Read input from stdin and send to process - :param name: - :param process: process to send input to - :return: - """ - global needPassword - debug ("starting reader for %s" % name ) - while not finished: - if needPassword: - needPassword = False - if sys.stdin.isatty(): - cred = getpass.getpass() - else: - cred = sys.stdin.readline().rstrip() - exe.stdin.write(cred + "\n") - -def debug(text): - if DEBUG: print '[DEBUG] ' + text - - -def error(text): - print '[ERROR] ' + text - sys.stdout.flush() - -def info(text): - print text - sys.stdout.flush() - - -def out(toStdErr, text) : - """ - Write to one of the system output channels. - This action does not add newlines. If you want that: write them yourself - :param toStdErr: flag set if stderr is to be the dest - :param text: text to write. - :return: - """ - if toStdErr: - sys.stderr.write(text) - else: - sys.stdout.write(text) - -def flush(toStdErr) : - """ - Flush the output stream - :param toStdErr: flag set if stderr is to be the dest - :return: - """ - if toStdErr: - sys.stderr.flush() - else: - sys.stdout.flush() - -def read(pipe, line): - """ - read a char, append to the listing if there is a char that is not \n - :param pipe: pipe to read from - :param line: line being built up - :return: (the potentially updated line, flag indicating newline reached) - """ - - c = pipe.read(1) - if c != "": - o = c.decode('utf-8') - if o != '\n': - line += o - return line, False - else: - return line, True - else: - return line, False - -def writePid(metadata_pid_file, process): - f = open(metadata_pid_file, 'w') - f.write(str(process.pid)) - f.close() - - http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/metadata_start.py ---------------------------------------------------------------------- diff --git a/src/bin/metadata_start.py b/src/bin/metadata_start.py deleted file mode 100755 index a111bed..0000000 --- a/src/bin/metadata_start.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import sys -import traceback - -import metadata_config as mc - -METADATA_LOG_OPTS="-Dmetadata.log.dir=%s -Dmetadata.log.file=application.log" -METADATA_COMMAND_OPTS="-Dmetadata.home=%s" -METADATA_CONFIG_OPTS="-Dmetadata.conf=%s" -DEFAULT_JVM_OPTS="-Xmx1024m" - -def main(): - - metadata_home = mc.metadataDir() - confdir = mc.dirMustExist(mc.confDir(metadata_home)) - mc.executeEnvSh(confdir) - logdir = mc.dirMustExist(mc.logDir(metadata_home)) - - #create sys property for conf dirs - jvm_opts_list = (METADATA_LOG_OPTS % logdir).split() - - cmd_opts = (METADATA_COMMAND_OPTS % metadata_home) - jvm_opts_list.extend(cmd_opts.split()) - - config_opts = (METADATA_CONFIG_OPTS % confdir) - jvm_opts_list.extend(config_opts.split()) - - default_jvm_opts = DEFAULT_JVM_OPTS - metadata_jvm_opts = os.environ.get(mc.METADATA_OPTS, default_jvm_opts) - jvm_opts_list.extend(metadata_jvm_opts.split()) - - #expand web app dir - web_app_dir = mc.webAppDir(metadata_home) - mc.expandWebApp(metadata_home) - - p = os.pathsep - metadata_classpath = confdir + p \ - + os.path.join(web_app_dir, "metadata", "WEB-INF", "classes" ) + p \ - + os.path.join(web_app_dir, "metadata", "WEB-INF", "lib", "*" ) + p \ - + os.path.join(metadata_home, "libext", "*") - - metadata_pid_file = mc.pidFile(metadata_home) - - if os.path.isfile(metadata_pid_file): - print "%s already exists, exiting" % metadata_pid_file - sys.exit() - - args = ["-app", os.path.join(web_app_dir, "metadata")] - args.extend(sys.argv[1:]) - - process = mc.java("org.apache.hadoop.metadata.Main", args, metadata_classpath, jvm_opts_list, logdir) - mc.writePid(metadata_pid_file, process) - - print "Metadata Server started!!!\n" - -if __name__ == '__main__': - try: - returncode = main() - except Exception as e: - print "Exception: %s " % str(e) - print traceback.format_exc() - returncode = -1 - - sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/metadata_stop.py ---------------------------------------------------------------------- diff --git a/src/bin/metadata_stop.py b/src/bin/metadata_stop.py deleted file mode 100755 index 192ebf2..0000000 --- a/src/bin/metadata_stop.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from signal import SIGTERM -import sys -import traceback - -import metadata_config as mc - -def main(): - - metadata_home = mc.metadataDir() - confdir = mc.dirMustExist(mc.confDir(metadata_home)) - mc.executeEnvSh(confdir) - piddir = mc.dirMustExist(mc.logDir(metadata_home)) - - metadata_pid_file = mc.pidFile(metadata_home) - - try: - pf = file(metadata_pid_file, 'r') - pid = int(pf.read().strip()) - pf.close() - except: - pid = None - - if not pid: - sys.stderr.write("No process ID file found. Server not running?\n") - return - - os.kill(pid, SIGTERM) - - # assuming kill worked since process check on windows is more involved... - if os.path.exists(metadata_pid_file): - os.remove(metadata_pid_file) - -if __name__ == '__main__': - try: - returncode = main() - except Exception as e: - print "Exception: %s " % str(e) - print traceback.format_exc() - returncode = -1 - - sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/dsl/DynamicTypedStruct.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/atlas/tools/dsl/DynamicTypedStruct.scala b/tools/src/main/scala/org/apache/atlas/tools/dsl/DynamicTypedStruct.scala new file mode 100755 index 0000000..7bc52e3 --- /dev/null +++ b/tools/src/main/scala/org/apache/atlas/tools/dsl/DynamicTypedStruct.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.tools.dsl + +import org.apache.atlas.typesystem.ITypedStruct +import org.apache.atlas.typesystem.types.{StructType, TypeSystem} + +import scala.language.dynamics + +class DynamicTypedStruct(val ts: ITypedStruct) extends Dynamic { + def selectDynamic(name: String) = ts.get(name) + + def updateDynamic(name: String)(value: Any) { + var value1 = value + if (value != null && value.isInstanceOf[DynamicTypedStruct]) { + value1 = value.asInstanceOf[DynamicTypedStruct].ts + } + ts.set(name, value1) + } + + def dataType = TypeSystem.getInstance().getDataType(classOf[StructType], ts.getTypeName) +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/dsl/package.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/atlas/tools/dsl/package.scala b/tools/src/main/scala/org/apache/atlas/tools/dsl/package.scala new file mode 100755 index 0000000..b7b7e0c --- /dev/null +++ b/tools/src/main/scala/org/apache/atlas/tools/dsl/package.scala @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.tools + +import org.apache.atlas.repository.memory.MemRepository +import org.apache.atlas.typesystem.json.{BigDecimalSerializer, BigIntegerSerializer, Serialization, TypedStructSerializer} +import org.apache.atlas.typesystem.persistence.StructInstance +import org.apache.atlas.typesystem.types._ +import org.apache.atlas.typesystem.{IStruct, ITypedStruct} +import org.json4s._ +import org.json4s.native.JsonMethods._ +import org.json4s.native.Serialization.{write => swrite} + +import scala.collection.JavaConversions._ +import scala.language.implicitConversions + +package object dsl { + + val defFormat = new DefaultFormats { + override protected def dateFormatter = TypeSystem.getInstance().getDateFormat; + + override val typeHints = NoTypeHints + } + + implicit val formats = defFormat + new TypedStructSerializer + + new BigDecimalSerializer + new BigIntegerSerializer + val BOOLEAN_TYPE = DataTypes.BOOLEAN_TYPE + val BYTE_TYPE = DataTypes.BYTE_TYPE + val SHORT_TYPE = DataTypes.SHORT_TYPE + val INT_TYPE = DataTypes.INT_TYPE + val LONG_TYPE = DataTypes.LONG_TYPE + val FLOAT_TYPE = DataTypes.FLOAT_TYPE + val DOUBLE_TYPE = DataTypes.DOUBLE_TYPE + val BIGINT_TYPE = DataTypes.BIGINTEGER_TYPE + val BIGDECIMAL_TYPE = DataTypes.BIGDECIMAL_TYPE + val DATE_TYPE = DataTypes.DATE_TYPE + val STRING_TYPE = DataTypes.STRING_TYPE + val ATTR_OPTIONAL = Multiplicity.OPTIONAL + val ATTR_REQUIRED = Multiplicity.REQUIRED + + def repo = new MemRepository(ts) + + def arrayType(dT: IDataType[_]) = ts.defineArrayType(dT) + + def mapType(kT: IDataType[_], vT: IDataType[_]) = ts.defineMapType(kT, vT) + + def attrDef(name: String, dT: IDataType[_], + m: Multiplicity = Multiplicity.OPTIONAL, + isComposite: Boolean = false, + reverseAttributeName: String = null) = { + require(name != null) + require(dT != null) + new AttributeDefinition(name, dT.getName, m, isComposite, reverseAttributeName) + } + + def listTypes = (ts.getTypeNames -- ts.getCoreTypes).sorted.toList.mkString("[", ",", "]") + + def ts = TypeSystem.getInstance + + def defineStructType(name: String, attrDef: AttributeDefinition*) = { + require(name != null) + ts.defineStructType(name, false, attrDef: _*) + } + + def createInstance(typeName: String, jsonStr: String)(implicit formats: Formats) = { + val j = parse(jsonStr) + assert(j.isInstanceOf[JObject]) + var j1 = j.asInstanceOf[JObject] + j1 = JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(typeName)) :: j1.obj) + new DynamicTypedStruct(Extraction.extract[StructInstance](j1)) + } + + def createInstance(typeName: String) = { + new DynamicTypedStruct( + ts.getDataType(classOf[StructType], typeName).asInstanceOf[IConstructableType[IStruct, ITypedStruct]].createInstance()) + } + + implicit def dynTypedStructToTypedStruct(s: DynamicTypedStruct) = s.ts + + implicit def dynTypedStructToJson(s: DynamicTypedStruct)(implicit formats: Formats) = { + Extraction.decompose(s.ts)(formats) + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/simpleserver/Main.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/atlas/tools/simpleserver/Main.scala b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/Main.scala new file mode 100755 index 0000000..7b98d77 --- /dev/null +++ b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/Main.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.tools.simpleserver + +import akka.actor.{ActorSystem, Props} +import akka.io.IO +import com.typesafe.config.ConfigFactory +import org.apache.atlas.repository.memory.MemRepository +import org.apache.atlas.typesystem.types.TypeSystem +import spray.can.Http + +/** + * A Simple Spray based server to test the TypeSystem and MemRepository. + * + * @example {{{ + * -- Using the [[ https://github.com/jakubroztocil/httpie Httpie tool]] + * + * http GET localhost:9140/listTypeNames + * pbpaste | http PUT localhost:9140/defineTypes + * http GET localhost:9140/typeDetails typeNames:='["Department", "Person", "Manager"]' + * + * pbpaste | http PUT localhost:9140/createInstance + * pbpaste | http GET localhost:9140/getInstance + * }}} + * + * - On the Mac, pbpaste makes available what is copied to clipboard. Copy contents of resources/sampleTypes.json + * - for createInstance resources/sampleInstance.json is an example + * - for getInstance send an Id back, you can copy the output from createInstance. + * + */ +object Main extends App { + val config = ConfigFactory.load() + val host = config.getString("http.host") + val port = config.getInt("http.port") + + implicit val system = ActorSystem("atlasservice") + + val typSys = TypeSystem.getInstance() + val memRepo = new MemRepository(typSys) + + val api = system.actorOf(Props(new RestInterface(typSys, memRepo)), "httpInterface") + IO(Http) ! Http.Bind(listener = api, interface = host, port = port) +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/simpleserver/MetadataActor.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/atlas/tools/simpleserver/MetadataActor.scala b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/MetadataActor.scala new file mode 100755 index 0000000..0429814 --- /dev/null +++ b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/MetadataActor.scala @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.tools.simpleserver + +import akka.actor._ +import akka.util.Timeout +import com.google.common.collect.ImmutableList +import org.apache.atlas.repository.memory.MemRepository +import org.apache.atlas.typesystem.json._ +import org.apache.atlas.typesystem.persistence.Id +import org.apache.atlas.typesystem.types._ +import org.apache.atlas.typesystem.{ITypedReferenceableInstance, TypesDef} +import org.json4s.{Formats, NoTypeHints} +import spray.httpx.Json4sSupport + +import scala.concurrent.duration._ + + +class MetadataActor(val typeSystem: TypeSystem, val memRepository: MemRepository) extends Actor with ActorLogging { + + import org.apache.atlas.tools.simpleserver.MetadataProtocol._ + + import scala.collection.JavaConversions._ + import scala.language.postfixOps + implicit val timeout = Timeout(5 seconds) + + + def receive = { + case ListTypeNames() => + sender ! TypeNames(typeSystem.getTypeNames.toList) + + case GetTypeDetails(typeNames) => + val typesDef = TypesSerialization.convertToTypesDef(typeSystem, (d: IDataType[_]) => typeNames.contains(d.getName)) + sender ! TypeDetails(typesDef) + + case DefineTypes(typesDef: TypesDef) => + typesDef.enumTypes.foreach(typeSystem.defineEnumType(_)) + + typeSystem.defineTypes(ImmutableList.copyOf(typesDef.structTypes.toArray), + ImmutableList.copyOf(typesDef.traitTypes.toArray), + ImmutableList.copyOf(typesDef.classTypes.toArray)) + + var newTypes: List[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]] = Nil + typesDef.traitTypes.foreach { tDef => + val nm = tDef.typeName + newTypes = newTypes :+ + typeSystem.getDataType(classOf[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]], nm) + } + typesDef.classTypes.foreach { tDef => + val nm = tDef.typeName + newTypes = newTypes :+ + typeSystem.getDataType(classOf[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]], nm) + } + + memRepository.defineTypes(newTypes) + + sender ! TypesCreated + + case CreateInstance(i) => + val r = memRepository.create(i) + sender ! InstanceCreated(r.getId) + + case GetInstance(id) => + val r = memRepository.get(id) + sender ! InstanceDetails(r) + } + +} + +object MetadataProtocol { + + case class ListTypeNames() + + case class TypeNames(typeNames: List[String]) + + case class GetTypeDetails(typeNames: List[String]) + + case class TypeDetails(types: TypesDef) + + case class DefineTypes(types: TypesDef) + + case class TypesCreated() + + case class CreateInstance(i: ITypedReferenceableInstance) + + case class InstanceCreated(id: Id) + + case class GetInstance(id: Id) + + case class InstanceDetails(i: ITypedReferenceableInstance) + +} + + +trait Json4sProtocol extends Json4sSupport { + val typeSystem: TypeSystem + val memRepository: MemRepository + + implicit def json4sFormats: Formats = + org.json4s.native.Serialization.formats(NoTypeHints) + new MultiplicitySerializer + + new TypedStructSerializer + + new TypedReferenceableInstanceSerializer + + new BigDecimalSerializer + new BigIntegerSerializer + new IdSerializer +} + http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/simpleserver/RestInterface.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/atlas/tools/simpleserver/RestInterface.scala b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/RestInterface.scala new file mode 100755 index 0000000..6a48165 --- /dev/null +++ b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/RestInterface.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.tools.simpleserver + +import akka.actor._ +import akka.util.Timeout +import org.apache.atlas.repository.memory.MemRepository +import org.apache.atlas.typesystem.persistence.Id +import org.apache.atlas.typesystem.types.TypeSystem +import org.apache.atlas.typesystem.{ITypedReferenceableInstance, TypesDef} +import spray.http.StatusCodes +import spray.routing._ + +import scala.concurrent.duration._ + +class Responder(val typeSystem: TypeSystem, val memRepository : MemRepository, + requestContext:RequestContext, mdSvc:ActorRef) extends Actor with Json4sProtocol with ActorLogging { + import org.apache.atlas.tools.simpleserver.MetadataProtocol._ + + def receive = { + + case typNames:TypeNames => + requestContext.complete(StatusCodes.OK, typNames) + self ! PoisonPill + + case tD:TypeDetails => + requestContext.complete(StatusCodes.OK, tD) + self ! PoisonPill + + case TypesCreated => + requestContext.complete(StatusCodes.OK) + self ! PoisonPill + + case InstanceCreated(id) => + requestContext.complete(StatusCodes.OK, id) + + case InstanceDetails(i) => + requestContext.complete(StatusCodes.OK, i) + } +} + +class RestInterface(val typeSystem: TypeSystem, val memRepository : MemRepository) extends HttpServiceActor +with RestApi { + def receive = runRoute(routes) +} + + +trait RestApi extends HttpService with Json4sProtocol with ActorLogging { actor: Actor => + import MetadataProtocol._ + + import scala.concurrent.ExecutionContext.Implicits.global + import scala.language.postfixOps + + val typeSystem : TypeSystem + val memRepository : MemRepository + + implicit val timeout = Timeout(10 seconds) + + import akka.pattern.{ask, pipe} + + val mdSvc = context.actorOf(Props(new MetadataActor(typeSystem, memRepository))) + + def routes: Route = + + path("listTypeNames") { + get { requestContext => + val responder: ActorRef = createResponder(requestContext) + + pipe(mdSvc.ask(ListTypeNames)) + + mdSvc.ask(ListTypeNames()).pipeTo(responder) + } + } ~ + path("typeDetails") { + get { + entity(as[GetTypeDetails]) { typeDetails => requestContext => + val responder = createResponder(requestContext) + mdSvc.ask(typeDetails).pipeTo(responder) + } + } + } ~ + path("defineTypes") { + put { + entity(as[TypesDef]) { typesDef => requestContext => + val responder = createResponder(requestContext) + mdSvc.ask(DefineTypes(typesDef)).pipeTo(responder) + } + } + } ~ + path("createInstance") { + put { + entity(as[ITypedReferenceableInstance]) { i => requestContext => + val responder = createResponder(requestContext) + mdSvc.ask(CreateInstance(i)).pipeTo(responder) + } + } + } ~ + path("getInstance") { + get { + entity(as[Id]) { id => requestContext => + val responder = createResponder(requestContext) + mdSvc.ask(GetInstance(id)).pipeTo(responder) + } + } + } + + def createResponder(requestContext:RequestContext) = { + context.actorOf(Props(new Responder(typeSystem, memRepository, requestContext, mdSvc))) + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftParser.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftParser.scala b/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftParser.scala new file mode 100755 index 0000000..67d4b37 --- /dev/null +++ b/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftParser.scala @@ -0,0 +1,664 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.tools.thrift + +import org.apache.atlas.MetadataException +import org.apache.atlas.typesystem.types.DataTypes + +import scala.util.parsing.combinator.lexical.StdLexical +import scala.util.parsing.combinator.syntactical.StandardTokenParsers +import scala.util.parsing.combinator.{ImplicitConversions, PackratParsers} +import scala.util.parsing.input.CharArrayReader._ + +object BASE_TYPES extends Enumeration { + val STRING = Value("string") + val BINARY = Value("binary") + val SLIST = Value("slist") + val BOOLEAN = Value("bool") + val BYTE = Value("byte") + val I16 = Value("i16") + val I32 = Value("i32") + val I64 = Value("i64") + val DOUBLE = Value("double") + + @throws[MetadataException] + def toPrimitiveTypeName(t : BASE_TYPES.Value) : String = t match { + case STRING => DataTypes.STRING_TYPE.getName + case SLIST => DataTypes.STRING_TYPE.getName + case BOOLEAN => DataTypes.BOOLEAN_TYPE.getName + case BYTE => DataTypes.BYTE_TYPE.getName + case I16 => DataTypes.SHORT_TYPE.getName + case I32 => DataTypes.INT_TYPE.getName + case I64 => DataTypes.LONG_TYPE.getName + case DOUBLE => DataTypes.DOUBLE_TYPE.getName + case _ => throw new MetadataException(s"Thrift BaseType ($t) not supported") + } +} + +object THRIFT_LANG extends Enumeration { + val CPP = Value("cpp") + val PHP = Value("php") + val PY = Value("py") + val PERL = Value("perl") + val RUBY = Value("ruby") + val SMLTK_CAT = Value("smalltalk.category") + val SMLTK_PRE = Value("smalltalk.prefix") + val JAVA = Value("java") + val COCOA = Value("cocoa") + val XSD = Value("xsd") + val CSHARP = Value("csharp") + val STAR = Value("*") + val OTHER = Value("") +} + +case class TypeAnnotation(name : String, value : String) +case class CPPType(name : String) +sealed trait FunctionType +case class VoidType() extends FunctionType +sealed trait FieldType extends FunctionType +case class IdentifierType(name : String) extends FieldType +case class BaseType(typ : BASE_TYPES.Value, typAnnotations :Option[List[TypeAnnotation]]) extends FieldType +sealed trait ContainerType extends FieldType { + def typAnnotations :Option[List[TypeAnnotation]] +} +case class MapType(keyType : FieldType, valueType : FieldType, + cppType : Option[CPPType], + typAnnotations :Option[List[TypeAnnotation]]) extends ContainerType +case class SetType(elemType : FieldType, + cppType : Option[CPPType], + typAnnotations :Option[List[TypeAnnotation]]) extends ContainerType +case class ListType(elemType : FieldType, + cppType : Option[CPPType], + typAnnotations :Option[List[TypeAnnotation]]) extends ContainerType + +sealed trait ConstValue +case class IntConstant(value : Int) extends ConstValue +case class DoubleConstant(value : Double) extends ConstValue +case class StringConstant(value : String) extends ConstValue +case class IdConstant(value : String) extends ConstValue +case class ConstantList(value : List[ConstValue]) extends ConstValue +case class ConstantValuePair(first : ConstValue, second : ConstValue) +case class ConstantMap(value : List[ConstantValuePair]) extends ConstValue + +case class ConstDef(fieldType : FieldType, id : String, value : ConstValue) + +case class TypeDef(name : String, fieldType : FieldType, + typAnnotations :Option[List[TypeAnnotation]]) +case class EnumValueDef(value : String, id : Option[IntConstant], typAnnotations :Option[List[TypeAnnotation]]) +case class EnumDef(name : String, enumValues : List[EnumValueDef], typAnnotations :Option[List[TypeAnnotation]]) + +case class SEnumDef(name : String, enumValues : List[String], typAnnotations :Option[List[TypeAnnotation]]) + +case class FieldDef(id : Option[IntConstant], requiredNess : Boolean, fieldType : FieldType, name : String, + fieldValue : Option[ConstValue], xsdOptional : Boolean, xsdNillable : Boolean, + xsdAttributes: Option[XsdAttributes], + typAnnotations :Option[List[TypeAnnotation]]) + +case class XsdAttributes(fields : List[FieldDef]) + +case class StructDef(name : String, xsdAll : Boolean, fields : List[FieldDef], + typAnnotations :Option[List[TypeAnnotation]]) + +case class UnionDef(val name : String, val xsdAll : Boolean, + val fields : List[FieldDef], + val typAnnotations :Option[List[TypeAnnotation]]) + +case class ExceptionDef(val name : String, + val fields : List[FieldDef], + val typAnnotations :Option[List[TypeAnnotation]]) + +case class FunctionDef(oneway : Boolean, returnType : FunctionType, name : String, parameters : List[FieldDef], + throwFields : Option[List[FieldDef]], typAnnotations :Option[List[TypeAnnotation]]) + +case class ServiceDef(name : String, superName : Option[String], functions : List[FunctionDef], + typAnnotations :Option[List[TypeAnnotation]]) + +case class IncludeDef(value : String) +case class CppIncludeDef(val value : String) +case class NamespaceDef(lang : THRIFT_LANG.Value, name : String, otherLang : Option[String] = None) + +case class ThriftDef(val includes : List[IncludeDef], + val cppIncludes : List[CppIncludeDef], + val namespaces : List[NamespaceDef], + val constants : List[ConstDef], + val typedefs : List[TypeDef], + val enums : List[EnumDef], + val senums : List[SEnumDef], + val structs : List[StructDef], + val unions : List[UnionDef], + val xceptions : List[ExceptionDef], + val services : List[ServiceDef]) { + + def this() = this(List(), List(), List(), List(), List(), List(), List(), + List(), List(), List(), List()) + + def this(a : IncludeDef) = this(a :: Nil, List(), List(), List(), List(), List(), List(), + List(), List(), List(), List()) + def this(a : CppIncludeDef) = this(List(), a :: Nil, List(), List(), List(), List(), List(), List(), + List(), List(), List()) + def this(a : NamespaceDef) = this(List(), List(), a :: Nil, List(), List(), List(), List(), List(), List(), + List(), List()) + def this(a : ConstDef) = this(List(), List(), List(), a :: Nil, List(), List(), List(), List(), List(), List(), + List()) + def this(a : TypeDef) = this(List(), List(), List(), List(), a :: Nil, List(), List(), List(), List(), List(), List()) + def this(a : EnumDef) = this(List(), List(), List(), List(), List(), a :: Nil, List(), List(), + List(), List(), List()) + def this(a : SEnumDef) = this(List(), List(), List(), List(), List(), List(), a :: Nil, List(), + List(), List(), List()) + def this(a : StructDef) = this(List(), List(), List(), List(), List(), List(), List(), a :: Nil, + List(), List(), List()) + def this(a : UnionDef) = this(List(), List(), List(), List(), List(), List(), List(), + List(), a :: Nil, List(), List()) + def this(a : ExceptionDef) = this(List(), List(), List(), List(), List(), List(), List(), + List(), List(), a :: Nil, List()) + def this(a : ServiceDef) = this(List(), List(), List(), List(), List(), List(), List(), + List(), List(), List(), a :: Nil) + + + def plus(a : IncludeDef) = ThriftDef(includes.+:(a), cppIncludes, namespaces, constants, typedefs, enums, senums, + structs, unions, xceptions, services) + def plus(a : CppIncludeDef) = ThriftDef(includes, cppIncludes.+:(a), namespaces, constants, typedefs, enums, senums, + structs, unions, xceptions, services) + def plus(a : NamespaceDef) = ThriftDef(includes, cppIncludes, namespaces.+:(a), constants, typedefs, enums, senums, + structs, unions, xceptions, services) + def plus(a : ConstDef) = ThriftDef(includes, cppIncludes, namespaces, constants.+:(a), typedefs, enums, senums, + structs, unions, xceptions, services) + def plus(a : TypeDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs.+:(a), enums, senums, + structs, unions, xceptions, services) + def plus(a : EnumDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums.+:(a), senums, + structs, unions, xceptions, services) + def plus(a : SEnumDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums.+:(a), + structs, unions, xceptions, services) + def plus(a : StructDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums, + structs.+:(a), unions, xceptions, services) + def plus(a : UnionDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums, + structs, unions.+:(a), xceptions, services) + def plus(a : ExceptionDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums, + structs, unions, xceptions.+:(a), services) + def plus(a : ServiceDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums, + structs, unions, xceptions, services.+:(a)) + def plus(a : ThriftDef) = ThriftDef(includes ::: a.includes, + cppIncludes ::: a.cppIncludes, + namespaces ::: a.namespaces, + constants ::: a.constants, + typedefs ::: a.typedefs, + enums ::: a.enums, + senums ::: a.senums, + structs ::: a.structs, + unions ::: a.unions, + xceptions ::: a.xceptions, + services ::: a.services) + + +} + +trait ThriftKeywords { + this : StandardTokenParsers => + + import scala.language.implicitConversions + + protected case class Keyword(str: String) + + protected implicit def asParser(k: Keyword): Parser[String] = k.str + + protected val LPAREN = Keyword("(") + protected val RPAREN = Keyword(")") + protected val EQ = Keyword("=") + protected val CPP_TYPE = Keyword("cpp_type") + protected val LIST = Keyword("list") + protected val LT = Keyword("<") + protected val GT = Keyword(">") + protected val SET = Keyword("set") + protected val MAP = Keyword("map") + protected val STRING = Keyword("string") + protected val BINARY = Keyword("binary") + protected val SLIST = Keyword("slist") + protected val BOOL = Keyword("bool") + protected val BYTE = Keyword("byte") + protected val I16 = Keyword("i16") + protected val I32 = Keyword("i32") + protected val I64 = Keyword("i64") + protected val DOUBLE = Keyword("double") + protected val VOID = Keyword("void") + protected val REQUIRED = Keyword("required") + protected val OPTIONAL = Keyword("optional") + protected val COLON = Keyword(":") + protected val THROWS = Keyword("throws") + protected val ONEWAY = Keyword("oneway") + protected val EXTENDS = Keyword("extends") + protected val SERVICE = Keyword("service") + protected val EXCEPTION = Keyword("exception") + protected val LBRACKET = Keyword("{") + protected val RBRACKET = Keyword("}") + protected val XSD_ATTRS = Keyword("xsd_attributes") + protected val XSD_NILBLE = Keyword("xsd_nillable") + protected val XSD_OPT = Keyword("xsd_optional") + protected val XSD_ALL = Keyword("xsd_all") + protected val UNION = Keyword("union") + protected val LSQBRACKET = Keyword("[") + protected val RSQBRACKET = Keyword("]") + protected val CONST = Keyword("const") + protected val STRUCT = Keyword("struct") + protected val SENUM = Keyword("senum") + protected val ENUM = Keyword("enum") + protected val COMMA = Keyword(",") + protected val SEMICOLON = Keyword(";") + protected val TYPEDEF = Keyword("typedef") + protected val INCLUDE = Keyword("include") + protected val CPP_INCL = Keyword("cpp_include") + protected val NAMESPACE = Keyword("namespace") + protected val STAR = Keyword("*") + protected val CPP_NS = Keyword("cpp_namespace") + protected val PHP_NS = Keyword("php_namespace") + protected val PY_NS = Keyword("py_module") + protected val PERL_NS = Keyword("perl_package") + protected val RUBY_NS = Keyword("ruby_namespace") + protected val SMLTK_CAT = Keyword("smalltalk_category") + protected val SMLTK_PRE = Keyword("smalltalk_prefix") + protected val JAVA_NS = Keyword("java_package") + protected val COCOA_NS = Keyword("cocoa_package") + protected val XSD_NS = Keyword("xsd_namespace") + protected val CSHARP_NS = Keyword("csharp_namespace") + + def isRequired(r : Option[String]) = r match { + case Some(REQUIRED) => true + case _ => false + } + + def isXsdOptional(r : Option[String]) = r match { + case Some(XSD_OPT) => true + case _ => false + } + + def isXsdNillable(r : Option[String]) = r match { + case Some(XSD_NILBLE) => true + case _ => false + } + + def isXsdAll(r : Option[String]) = r match { + case Some(XSD_ALL) => true + case _ => false + } + + def isOneWay(r : Option[String]) = r match { + case Some(ONEWAY) => true + case _ => false + } + +} + +trait ThriftTypeRules extends ThriftKeywords { + this : StandardTokenParsers => + + def containterType : Parser[ContainerType] = mapType | setType | listType + + def setType = SET ~ cppType.? ~ LT ~ fieldType ~ GT ~ typeAnnotations.? ^^ { + case s ~ ct ~ lt ~ t ~ gt ~ tA => SetType(t, ct, tA) + } + + def listType = LIST ~ LT ~ fieldType ~ GT ~ cppType.? ~ typeAnnotations.? ^^ { + case l ~ lt ~ t ~ gt ~ ct ~ tA => ListType(t, ct, tA) + } + + def mapType = MAP ~ cppType.? ~ LT ~ fieldType ~ COMMA ~ fieldType ~ GT ~ typeAnnotations.? ^^ { + case s ~ ct ~ lt ~ kt ~ c ~ vt ~ gt ~ tA => MapType(kt, vt, ct, tA) + } + + def cppType : Parser[CPPType] = CPP_TYPE ~ stringLit ^^ { case c ~ s => CPPType(s)} + + def fieldType: Parser[FieldType] = ident ^^ {case i => IdentifierType(i)} | + baseType | + containterType + + def baseType : Parser[BaseType] = simpleBaseType ~ typeAnnotations.? ^^ { case s ~ t => BaseType(s, t)} + + def simpleBaseType : Parser[BASE_TYPES.Value] = STRING ^^^ BASE_TYPES.STRING | + BINARY ^^^ BASE_TYPES.BINARY | + SLIST ^^^ BASE_TYPES.SLIST | + BOOL ^^^ BASE_TYPES.BOOLEAN | + BYTE ^^^ BASE_TYPES.BYTE | + I16 ^^^ BASE_TYPES.I16 | + I32 ^^^ BASE_TYPES.I32 | + I64 ^^^ BASE_TYPES.I64 | + DOUBLE ^^^ BASE_TYPES.DOUBLE + + def typeAnnotations : Parser[List[TypeAnnotation]] = + LPAREN ~ typeAnnotation.* ~ RPAREN ^^ { case l ~ t ~ r => t.toList} + + def typeAnnotation : Parser[TypeAnnotation] = + (ident ~ EQ ~ stringLit ~ commaOrSemicolon.?) ^^ { case i ~ e ~ s ~ c => TypeAnnotation(i,s)} + + def commaOrSemicolon : Parser[String] = COMMA | SEMICOLON + +} + +/** + * @todo extract Constant Rules into this Trait. This requires moving `hexConstant` here. But how to specify + * type of `HexConstant`, it is a Path dependent Type tied to lexical member of ThriftParser. + */ +trait ThriftConstantRules extends ThriftKeywords { + this: StandardTokenParsers => + + // def parseDouble(s: String) = try { Some(s.toDouble) } catch { case _ : Throwable => None } + // + // def constValue : Parser[ConstValue] = numericLit ^^ { + // case n => parseDouble(n) match { + // case Some(d) => DoubleConstant(d) + // case _ => IntConstant(n.toInt) + // } + // } | + // hexConstant ^^ { case h => IntConstant(Integer.parseInt(h, 16))} | + // stringLit ^^ { case s => StringConstant(s)} | + // ident ^^ { case i => IdConstant(i)} | + // constList | + // constMap + // + // def constValuePair = constValue ~ COLON ~ constValue ~ commaOrSemicolon.? ^^ { + // case k ~ c ~ v ~ cs => ConstantValuePair(k,v) + // } + // + // def constList = LSQBRACKET ~ (constValue <~ commaOrSemicolon).* ~ RSQBRACKET ^^ { + // case l ~ vs ~ r => ConstantList(vs) + // } + // + // def constMap = LBRACKET ~ constValuePair.* ~ RBRACKET ^^ { + // case l ~ ps ~ r => ConstantMap(ps) + // } +} + +/** + * A Parser for Thrift definition scripts. + * Based on [[https://github.com/twitter/commons/blob/master/src/antlr/twitter/thrift/descriptors/AntlrThrift.g]]. + * Definition is parsed into a [[org.apache.atlas.tools.thrift.ThriftDef ThriftDef]] structure. + * + * @example {{{ + * var p = new ThriftParser + * var td : Option[ThriftDef] = p("""include "share/fb303/if/fb303.thrift" + * namespace java org.apache.hadoop.hive.metastore.api + * namespace php metastore + * namespace cpp Apache.Hadoop.Hive + * \""") + * }}} + * + * @todo doesn't traverse includes directives. Includes are parsed into + * [[org.apache.atlas.tools.thrift.IncludeDef IncludeDef]] structures + * but are not traversed. + * @todo mixing in [[scala.util.parsing.combinator.PackratParsers PackratParsers]] is a placeholder. Need to + * change specific grammar rules to `lazy val` and `Parser[Elem]` to `PackratParser[Elem]`. Will do based on + * performance analysis. + * @todo Error reporting + */ +class ThriftParser extends StandardTokenParsers with ThriftKeywords with ThriftTypeRules with PackratParsers { + + import scala.language.higherKinds + + private val reservedWordsDelims : Seq[String] = + this + .getClass + .getMethods + .filter(_.getReturnType == classOf[Keyword]) + .map(_.invoke(this).asInstanceOf[Keyword].str) + + private val (thriftreservedWords : Seq[String], thriftdelims : Seq[String]) = + reservedWordsDelims.partition(s => s.charAt(0).isLetter) + + override val lexical = new ThriftLexer(thriftreservedWords, thriftdelims) + + import lexical.HexConstant + /** A parser which matches a hex constant */ + def hexConstant: Parser[String] = + elem("string literal", _.isInstanceOf[HexConstant]) ^^ (_.chars) + + def apply(input: String): Option[ThriftDef] = { + phrase(program)(new lexical.Scanner(input)) match { + case Success(r, x) => Some(r) + case Failure(m, x) => { + None + } + case Error(m, x) => { + None + } + } + } + + def program = headers ~ definitions ^^ { case h ~ d => h plus d} + + def headers = header.* ^^ { case l => l.foldRight(new ThriftDef)((a,t) => t plus a)} + + def header = INCLUDE ~> stringLit ^^ { case s => new ThriftDef(IncludeDef(s))} | + CPP_INCL ~> stringLit ^^ { case s => new ThriftDef(CppIncludeDef(s))} | + NAMESPACE ~ ident ~ ident ^^ { case ns ~ t ~ n => new ThriftDef(NamespaceDef(THRIFT_LANG.OTHER, n, Some(t)))} | + NAMESPACE ~ STAR ~ ident ^^ { case ns ~ s ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.STAR, i))} | + CPP_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.CPP, i))} | + PHP_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.PHP, i))} | + PY_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.PY, i))} | + PERL_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.PERL, i))} | + RUBY_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.RUBY, i))} | + SMLTK_CAT ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.SMLTK_CAT, i))} | + SMLTK_PRE ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.SMLTK_PRE, i))} | + JAVA_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.JAVA, i))} | + COCOA_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.COCOA, i))} | + XSD_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.XSD, i))} | + CSHARP_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.CSHARP, i))} + + def definitions : Parser[ThriftDef] = definition.* ^^ { + case l => l.foldRight(new ThriftDef)((a,t) => t plus a) + } + + def definition : Parser[ThriftDef] = const ^^ { case c => new ThriftDef(c)} | + typeDefinition | + service ^^ { case s => new ThriftDef(s)} + + + def typeDefinition : Parser[ThriftDef] = (typedef ^^ {case t => new ThriftDef(t)} | + enum ^^ {case e => new ThriftDef(e)} | + senum ^^ {case e => new ThriftDef(e)} | + struct ^^ {case e => new ThriftDef(e)} | + union ^^ {case e => new ThriftDef(e)} | + xception ^^ {case e => new ThriftDef(e)} + ) + + def typedef : Parser[TypeDef] = TYPEDEF ~ fieldType ~ ident ~ typeAnnotations.? ^^ { + case t ~ f ~ i ~ tA => TypeDef(i, f, tA) + } + + def enum : Parser[EnumDef] = ENUM ~ ident ~ LBRACKET ~ enumDef.* ~ RBRACKET ~ typeAnnotations.? ^^ { + case e ~ i ~ l ~ ed ~ r ~ t => EnumDef(i, ed.toList, t) + } + + def enumDef : Parser[EnumValueDef] = ident ~ EQ ~ numericLit ~ typeAnnotations.? ~ commaOrSemicolon.? ^^ { + case i ~ e ~ n ~ t ~ c => EnumValueDef(i, Some(IntConstant(n.toInt)), t) + } + + def senum : Parser[SEnumDef] = SENUM ~ ident ~ LBRACKET ~ senumDef.* ~ RBRACKET ~ typeAnnotations.? ^^ { + case se ~ i ~ l ~ sed ~ r ~ t => SEnumDef(i, sed.toList, t) + } + + def senumDef : Parser[String] = stringLit <~ commaOrSemicolon.? + + def service : Parser[ServiceDef] = SERVICE ~ ident ~ extnds.? ~ LBRACKET ~ function.* ~ + RBRACKET ~ typeAnnotations.? ^^ { + case s ~ i ~ e ~ lb ~ fs ~ rb ~ tA => ServiceDef(i, e, fs, tA) + } + + def extnds : Parser[String] = EXTENDS ~> ident + + def function : Parser[FunctionDef] = ONEWAY.? ~ functionType ~ ident ~ LPAREN ~ field.* ~ RPAREN ~ throwz.? ~ + typeAnnotations.? ~ commaOrSemicolon.? ^^ { + case o ~ fT ~ i ~ lp ~ fs ~ rp ~ th ~ tA ~ cS => FunctionDef(isOneWay(o), fT, i, fs, th, tA) + } + + def throwz : Parser[List[FieldDef]] = THROWS ~ LPAREN ~ field.* ~ RPAREN ^^ { + case t ~ l ~ fs ~ r => fs.toList + } + + def functionType : Parser[FunctionType] = VOID ^^^ VoidType() | fieldType + + def xception : Parser[ExceptionDef] = EXCEPTION ~ ident ~ LBRACKET ~ field.* ~ RBRACKET ~ typeAnnotations.? ^^ { + case s ~ i ~ lb ~ fs ~ rb ~ tA => ExceptionDef(i, fs.toList, tA) + } + + def union : Parser[UnionDef] = UNION ~ ident ~ XSD_ALL.? ~ LBRACKET ~ field.* ~ RBRACKET ~ typeAnnotations.? ^^ { + case s ~ i ~ xA ~ lb ~ fs ~ rb ~ tA => UnionDef(i, isXsdAll(xA), fs.toList, tA) + } + + def struct : Parser[StructDef] = STRUCT ~ ident ~ XSD_ALL.? ~ LBRACKET ~ field.* ~ RBRACKET ~ typeAnnotations.? ^^ { + case s ~ i ~ xA ~ lb ~ fs ~ rb ~ tA => StructDef(i, isXsdAll(xA), fs.toList, tA) + } + + def field : Parser[FieldDef] = fieldIdentifier.? ~ fieldRequiredness.? ~ fieldType ~ ident ~ fieldValue.? ~ + XSD_OPT.? ~ XSD_NILBLE.? ~ xsdAttributes.? ~ typeAnnotations.? ~ commaOrSemicolon.? ^^ { + case fi ~ fr ~ ft ~id ~ fv ~ xo ~ xn ~ xa ~ tA ~ cS => FieldDef( + fi, + isRequired(fr), + ft, + id, + fv, + isXsdOptional(xo), + isXsdNillable(xn), + xa, + tA + ) + } + + def xsdAttributes : Parser[XsdAttributes] = XSD_ATTRS ~ LBRACKET ~ field.* ~ RBRACKET ^^ { + case x ~ l ~ f ~ r => XsdAttributes(f) + } + + def fieldValue = EQ ~> constValue + + def fieldRequiredness : Parser[String] = REQUIRED | OPTIONAL + + def fieldIdentifier : Parser[IntConstant] = numericLit <~ COLON ^^ { + case n => IntConstant(n.toInt) + } + + def const : Parser[ConstDef] = CONST ~ fieldType ~ ident ~ EQ ~ constValue ~ commaOrSemicolon.? ^^ { + case c ~ fT ~ i ~ e ~ cV ~ cS => ConstDef(fT, i, cV) + } + + def parseDouble(s: String) = try { Some(s.toDouble) } catch { case _ : Throwable => None } + + def constValue : Parser[ConstValue] = numericLit ^^ { + case n => parseDouble(n) match { + case Some(d) => DoubleConstant(d) + case _ => IntConstant(n.toInt) + } + } | + hexConstant ^^ { case h => IntConstant(Integer.parseInt(h, 16))} | + stringLit ^^ { case s => StringConstant(s)} | + ident ^^ { case i => IdConstant(i)} | + constList | + constMap + + def constValuePair = constValue ~ COLON ~ constValue ~ commaOrSemicolon.? ^^ { + case k ~ c ~ v ~ cs => ConstantValuePair(k,v) + } + + def constList = LSQBRACKET ~ (constValue <~ commaOrSemicolon).* ~ RSQBRACKET ^^ { + case l ~ vs ~ r => ConstantList(vs) + } + + def constMap = LBRACKET ~ constValuePair.* ~ RBRACKET ^^ { + case l ~ ps ~ r => ConstantMap(ps) + } +} + +class ThriftLexer(val keywords: Seq[String], val delims : Seq[String]) extends StdLexical with ImplicitConversions { + + case class HexConstant(chars: String) extends Token { + override def toString = chars + } + + case class StIdentifier(chars: String) extends Token { + override def toString = chars + } + + reserved ++= keywords + + delimiters ++= delims + + override lazy val token: Parser[Token] = + ( intConstant ^^ NumericLit + | hexConstant ^^ HexConstant + | dubConstant ^^ NumericLit + | identifier ^^ processIdent + | st_identifier ^^ StIdentifier + | string ^^ StringLit + | EofCh ^^^ EOF + | '\'' ~> failure("unclosed string literal") + | '"' ~> failure("unclosed string literal") + | delim + | failure("illegal character") + ) + + override def identChar = letter | elem('_') + + def identifier = identChar ~ (identChar | digit | '.' ).* ^^ + { case first ~ rest => (first :: rest).mkString } + + def st_identChar = letter | elem('-') + def st_identifier = st_identChar ~ (st_identChar | digit | '.' | '_').* ^^ + { case first ~ rest => (first :: rest).mkString("")} + + override def whitespace: Parser[Any] = + ( whitespaceChar + | '/' ~ '*' ~ comment + | '/' ~ '/' ~ chrExcept(EofCh, '\n').* + | '#' ~ chrExcept(EofCh, '\n').* + | '/' ~ '*' ~ failure("unclosed comment") + ).* + + protected override def comment: Parser[Any] = ( + commentChar.* ~ '*' ~ '/' + ) + + protected def commentChar = chrExcept(EofCh, '*') | '*' ~ not('/') + + def string = '\"' ~> chrExcept('\"', '\n', EofCh).* <~ '\"' ^^ { _ mkString "" } | + '\'' ~> chrExcept('\'', '\n', EofCh).* <~ '\'' ^^ { _ mkString "" } + + def zero: Parser[String] = '0' ^^^ "0" + def nonzero = elem("nonzero digit", d => d.isDigit && d != '0') + def sign = elem("sign character", d => d == '-' || d == '+') + def exponent = elem("exponent character", d => d == 'e' || d == 'E') + + + def intConstant = opt(sign) ~> zero | intList + def intList = opt(sign) ~ nonzero ~ rep(digit) ^^ {case s ~ x ~ y => (optString("", s) :: x :: y) mkString ""} + def fracPart = '.' ~> rep(digit) ^^ { "." + _ mkString "" } + def expPart = exponent ~ opt(sign) ~ rep1(digit) ^^ { case e ~ s ~ d => + e.toString + optString("", s) + d.mkString("") + } + + def dubConstant = opt(sign) ~ digit.* ~ fracPart ~ opt(expPart) ^^ { case s ~ i ~ f ~ e => + optString("", s) + i + f + optString("", e) + } + + val hexDigits = Set[Char]() ++ "0123456789abcdefABCDEF".toArray + def hexDigit = elem("hex digit", hexDigits.contains(_)) + + def hexConstant = '0' ~> 'x' ~> hexDigit.+ ^^ {case h => h.mkString("")} + + + private def optString[A](pre: String, a: Option[A]) = a match { + case Some(x) => pre + x.toString + case None => "" + } +}
