Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6e716c6d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6e716c6d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6e716c6d Branch: refs/heads/trunk Commit: 6e716c6da41900950e32a5549b3bb1e858ecad18 Parents: c0765ed 6f360b6 Author: Robert Stupp <sn...@snazy.de> Authored: Thu Jan 5 22:20:31 2017 +0100 Committer: Robert Stupp <sn...@snazy.de> Committed: Thu Jan 5 22:20:31 2017 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + NEWS.txt | 3 ++ doc/cql3/CQL.textile | 2 +- lib/jsr223/clojure/README.txt | 8 --- lib/jsr223/groovy/README.txt | 35 ------------- lib/jsr223/jaskell/README.txt | 5 -- lib/jsr223/jruby/README.txt | 54 -------------------- lib/jsr223/jython/README.txt | 33 ------------ lib/jsr223/scala/README.txt | 37 -------------- .../cql3/functions/ScriptBasedUDFunction.java | 30 +++++------ .../cql3/validation/entities/UFScriptTest.java | 3 +- .../validation/entities/UFSecurityTest.java | 12 ++++- 12 files changed, 29 insertions(+), 194 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 666a771,b41313d..77a310d --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,23 -1,5 +1,24 @@@ -2.2.9 +3.0.11 + * Fixed flacky SSTableRewriterTest: check file counts before calling validateCFS (CASSANDRA-12348) + * Fix deserialization of 2.x DeletedCells (CASSANDRA-12620) + * Add parent repair session id to anticompaction log message (CASSANDRA-12186) + * Improve contention handling on failure to acquire MV lock for streaming and hints (CASSANDRA-12905) + * Fix DELETE and UPDATE queries with empty IN restrictions (CASSANDRA-12829) + * Mark MVs as built after successful bootstrap (CASSANDRA-12984) + * Estimated TS drop-time histogram updated with Cell.NO_DELETION_TIME (CASSANDRA-13040) + * Nodetool compactionstats fails with NullPointerException (CASSANDRA-13021) + * Thread local pools never cleaned up (CASSANDRA-13033) + * Set RPC_READY to false when draining or if a node is marked as shutdown (CASSANDRA-12781) + * CQL often queries static columns unnecessarily (CASSANDRA-12768) + * Make sure sstables only get committed when it's safe to discard commit log records (CASSANDRA-12956) + * Reject default_time_to_live option when creating or altering MVs (CASSANDRA-12868) + * Nodetool should use a more sane max heap size (CASSANDRA-12739) + * LocalToken ensures token values are cloned on heap (CASSANDRA-12651) + * AnticompactionRequestSerializer serializedSize is incorrect (CASSANDRA-12934) + * Prevent reloading of logback.xml from UDF sandbox (CASSANDRA-12535) + * Reenable HeapPool (CASSANDRA-12900) +Merged from 2.2: + * Remove support for non-JavaScript UDFs (CASSANDRA-12883) * Fix DynamicEndpointSnitch noop in multi-datacenter situations (CASSANDRA-13074) * cqlsh copy-from: encode column names to avoid primary key parsing errors (CASSANDRA-12909) * Temporarily fix bug that creates commit log when running offline tools (CASSANDRA-8616) http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/NEWS.txt ---------------------------------------------------------------------- diff --cc NEWS.txt index 32b5084,37949a1..b4e6551 --- a/NEWS.txt +++ b/NEWS.txt @@@ -13,77 -13,20 +13,80 @@@ restore snapshots created with the prev 'sstableloader' tool. You can upgrade the file format of your snapshots using the provided 'sstableupgrade' tool. -2.2.9 +3.0.11 ===== +Upgrading +--------- + - Nothing specific to this release, but please see previous versions upgrading section, + especially if you are upgrading from 2.2. + - Specifying the default_time_to_live option when creating or altering a + materialized view was erroneously accepted (and ignored). It is now + properly rejected. ++ - Only Java and JavaScript are now supported UDF languages. ++ The sandbox in 3.0 already prevented the use of script languages except Java ++ and JavaScript. + +3.0.10 +===== + +Upgrading +--------- + - memtable_allocation_type: offheap_buffers is no longer allowed to be specified in the 3.0 series. + This was an oversight that can cause segfaults. Offheap was re-introduced in 3.4 see CASSANDRA-11039 + and CASSANDRA-9472 for details. + +3.0.9 +===== + +Upgrading +--------- + - The ReversedType behaviour has been corrected for clustering columns of + BYTES type containing empty value. Scrub should be run on the existing + SSTables containing a descending clustering column of BYTES type to correct + their ordering. See CASSANDRA-12127 for more details. + +3.0.8 +===== + +Upgrading +--------- + - Ec2MultiRegionSnitch will no longer automatically set broadcast_rpc_address + to the public instance IP if this property is defined on cassandra.yaml. + +3.0.7 +===== + +Upgrading +--------- + - A maximum size for SSTables values has been introduced, to prevent out of memory + exceptions when reading corrupt SSTables. This maximum size can be set via + max_value_size_in_mb in cassandra.yaml. The default is 256MB, which matches the default + value of native_transport_max_frame_size_in_mb. SSTables will be considered corrupt if + they contain values whose size exceeds this limit. See CASSANDRA-9530 for more details. + Deprecation ----------- + - DateTieredCompactionStrategy has been deprecated - new tables should use + TimeWindowCompactionStrategy. Note that migrating an existing DTCS-table to TWCS might + cause increased compaction load for a while after the migration so make sure you run + tests before migrating. Read CASSANDRA-9666 for background on this. -(See note about the new feature User-Defined-Functions in 2.2.0.) +New features +------------ + - TimeWindowCompactionStrategy has been added. This has proven to be a better approach + to time series compaction and new tables should use this instead of DTCS. See + CASSANDRA-9666 for details. -Since the security manager added in 3.0 only allows Java and JavaScript -UDFs to be run, UDFs for other languages are deprecated and support for -non-Java and non-JavaScript UDFs is deprecated in 2.2 and has been removed -in version 3.0.11. +3.0.6 +===== + +New features +------------ + - JSON timestamps are now in UTC and contain the timezone information, see + CASSANDRA-11137 for more details. -2.2.8 +3.0.5 ===== Upgrading http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/doc/cql3/CQL.textile ---------------------------------------------------------------------- diff --cc doc/cql3/CQL.textile index 2a37452,af584d0..2544878 --- a/doc/cql3/CQL.textile +++ b/doc/cql3/CQL.textile @@@ -1977,7 -1910,7 +1977,7 @@@ SELECT AVG(players) FROM plays h2(#udfs). User-Defined Functions --User-defined functions allow execution of user-provided code in Cassandra. By default, Cassandra supports defining functions in _Java_ and _JavaScript_. Support for other JSR 223 compliant scripting languages (such as Python, Ruby, and Scala) can be added by adding a JAR to the classpath. ++User-defined functions allow execution of user-provided code in Cassandra. By default, Cassandra supports defining functions in _Java_ and _JavaScript_. Support for other JSR 223 compliant scripting languages (such as Python, Ruby, and Scala) has been removed in 3.0.11. UDFs are part of the Cassandra schema. As such, they are automatically propagated to all nodes in the cluster. http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/clojure/README.txt ---------------------------------------------------------------------- diff --cc lib/jsr223/clojure/README.txt index 7ed7551,7ed7551..0000000 deleted file mode 100644,100644 --- a/lib/jsr223/clojure/README.txt +++ /dev/null @@@ -1,8 -1,8 +1,0 @@@ --Apache Cassandra User-Defined-Functions JSR 223 scripting --========================================================= -- --Unfortunately the JSR-223 support provided by the project https://github.com/ato/clojure-jsr223 --and the related ones do not provide compileable script support. -- --The JSR-223 javax.script.Compilable implementation takes source file names or readers but not script sources --as all other JSR-223 implementations do. http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/groovy/README.txt ---------------------------------------------------------------------- diff --cc lib/jsr223/groovy/README.txt index 09fef93,09fef93..0000000 deleted file mode 100644,100644 --- a/lib/jsr223/groovy/README.txt +++ /dev/null @@@ -1,35 -1,35 +1,0 @@@ --Apache Cassandra User-Defined-Functions JSR 223 scripting --========================================================= -- --Using JSR-223 capable Groovy -- --Tested with version 2.3.6 -- --Installation -------------- -- --1. Download Groovy binary release --2. Unpack the downloaded archive into a temporary directory --3. Copy the jar groovy-all-2.3.6-indy.jar from the Groovy embeddable directory to $CASSANDRA_HOME/lib/jsr223/groovy -- "indy" means "invokedynamic" and is a JVM instruction for scripting languages new to Java 7. --4. Restart your Cassandra daemon if it's already running -- --Cassandra log should contain a line like this: -- INFO 10:49:45 Found scripting engine Groovy Scripting Engine 2.0 - Groovy 2.3.6 - language names: [groovy, Groovy] --Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below). -- --Smoke Test ------------ -- --To test Groovy functionality, open cqlsh and execute the following command: -- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE groovy AS 'return "foo";' ; -- --If you get the error -- code=2200 [Invalid query] message="Invalid language groovy for 'foobar'" --Groovy for Apache Cassandra has not been installed correctly. -- --Notes / Java7 invokedynamic ----------------------------- -- --Groovy provides jars that support invokedynamic bytecode instruction. These jars are whose ending with --"-indy.jar". http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jaskell/README.txt ---------------------------------------------------------------------- diff --cc lib/jsr223/jaskell/README.txt index 53e942e,53e942e..0000000 deleted file mode 100644,100644 --- a/lib/jsr223/jaskell/README.txt +++ /dev/null @@@ -1,5 -1,5 +1,0 @@@ --Apache Cassandra User-Defined-Functions JSR 223 scripting --========================================================= -- --Unfortunately Jaskell JSR-223 support is quite old and the Jaskell engine seems to be quite --unsupported. If you find a solution, please open a ticket at Apache Cassandra JIRA. http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jruby/README.txt ---------------------------------------------------------------------- diff --cc lib/jsr223/jruby/README.txt index cbc12dc,cbc12dc..0000000 deleted file mode 100644,100644 --- a/lib/jsr223/jruby/README.txt +++ /dev/null @@@ -1,54 -1,54 +1,0 @@@ --Apache Cassandra User-Defined-Functions JSR 223 scripting --========================================================= -- --Using JSR-223 capable JRuby -- --Tested with version 1.7.15 -- --Installation -------------- -- --1. Download JRuby binary release --2. Unpack the downloaded archive into a temporary directory --3. Copy everything from the JRuby lib directory to $CASSANDRA_HOME/lib/jsr223/jruby --4. Restart your Cassandra daemon if it's already running -- --Cassandra log should contain a line like this: -- INFO 10:29:03 Found scripting engine JSR 223 JRuby Engine 1.7.15 - ruby jruby 1.7.15 - language names: [ruby, jruby] --Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below). -- -- --Smoke Test ------------ -- --To test JRuby functionality, open cqlsh and execute the following command: -- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE ruby AS 'return "foo";' ; -- --If you get the error -- code=2200 [Invalid query] message="Invalid language ruby for 'foobar'" --JRuby for Apache Cassandra has not been installed correctly. -- -- --Ruby require/include ---------------------- -- --You can use Ruby require and include in your scripts as in the following example: -- -- --CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE ruby AS ' --require "bigdecimal" --require "bigdecimal/math" -- --include BigMath -- --a = BigDecimal((PI(100)/2).to_s) -- --return "foo " + a.to_s; --' ; -- -- --Notes / Java7 invokedynamic ----------------------------- -- --See JRuby wiki pages https://github.com/jruby/jruby/wiki/ConfiguringJRuby and --https://github.com/jruby/jruby/wiki/PerformanceTuning for more information and optimization tips. http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jython/README.txt ---------------------------------------------------------------------- diff --cc lib/jsr223/jython/README.txt index bef3c83,bef3c83..0000000 deleted file mode 100644,100644 --- a/lib/jsr223/jython/README.txt +++ /dev/null @@@ -1,33 -1,33 +1,0 @@@ --Apache Cassandra User-Defined-Functions JSR 223 scripting --========================================================= -- --Using JSR-223 capable Jython -- --Tested with version 2.3.5 -- --Installation -------------- -- --1. Download Jython binary release --2. Unpack the downloaded archive into a temporary directory --3. Copy the jar jython.jar from the Jython directory to $CASSANDRA_HOME/lib/jsr223/jython --4. Restart your Cassandra daemon if it's already running -- --Cassandra log should contain a line like this: -- INFO 10:58:18 Found scripting engine jython 2.5.3 - python 2.5 - language names: [python, jython] --Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below). -- --Smoke Test ------------ -- --To test Jython functionality, open cqlsh and execute the following command: -- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE python AS '''foo''' ; -- --If you get the error -- code=2200 [Invalid query] message="Invalid language python for 'foobar'" --Jython for Apache Cassandra has not been installed correctly. -- --Notes / Java7 invokedynamic ----------------------------- -- --Jython currently targets Java6 only. They want to switch to Java7 + invokedynamic in Jython 3. http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/scala/README.txt ---------------------------------------------------------------------- diff --cc lib/jsr223/scala/README.txt index 7f5d6fe,7f5d6fe..0000000 deleted file mode 100644,100644 --- a/lib/jsr223/scala/README.txt +++ /dev/null @@@ -1,37 -1,37 +1,0 @@@ --Apache Cassandra User-Defined-Functions JSR 223 scripting --========================================================= -- --Using JSR-223 capable Scala -- --Tested with version 2.11.2 -- --Installation -------------- -- --1. Download Scala binary release --2. Unpack the downloaded archive into a temporary directory --3. Copy the following jars from the Scala lib directory to $CASSANDRA_HOME/lib/jsr223/scala -- scala-compiler.jar -- scala-library.jar -- scala-reflect.jar --4. Restart your Cassandra daemon if it's already running -- --Cassandra log should contain a line like this: -- INFO 11:42:35 Found scripting engine Scala Interpreter 1.0 - Scala version 2.11.2 - language names: [scala] --Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below). -- --Smoke Test ------------ -- --To test Scala functionality, open cqlsh and execute the following command: -- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE scala AS 'return "foo";' ; -- --If you get the error -- code=2200 [Invalid query] message="Invalid language scala for 'foobar'" --Scala for Apache Cassandra has not been installed correctly. -- --Notes / Java7 invokedynamic ----------------------------- -- --Scala 2.10 has Java6 support only. 2.11 has experimental invokedynamic support (use at your own risk!). --2.12 introduces an upgrade directly to Java8 - see https://stackoverflow.com/questions/14285894/advantages-of-scala-emitting-bytecode-for-the-jvm-1-7 http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java index 8743a20,0000000..47deafa mode 100644,000000..100644 --- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java +++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java @@@ -1,246 -1,0 +1,240 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.cql3.functions; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.*; +import java.nio.ByteBuffer; +import java.security.*; +import java.security.cert.Certificate; +import java.util.*; +import java.util.concurrent.ExecutorService; +import javax.script.*; + ++import jdk.nashorn.api.scripting.ClassFilter; ++import jdk.nashorn.api.scripting.NashornScriptEngine; ++import jdk.nashorn.api.scripting.NashornScriptEngineFactory; +import org.apache.cassandra.concurrent.NamedThreadFactory; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.exceptions.InvalidRequestException; + +final class ScriptBasedUDFunction extends UDFunction +{ - static final Map<String, Compilable> scriptEngines = new HashMap<>(); - + private static final ProtectionDomain protectionDomain; + private static final AccessControlContext accessControlContext; + + // + // For scripted UDFs we have to rely on the security mechanisms of the scripting engine and + // SecurityManager - especially SecurityManager.checkPackageAccess(). Unlike Java-UDFs, strict checking + // of class access via the UDF class loader is not possible, since e.g. Nashorn builds its own class loader + // (jdk.nashorn.internal.runtime.ScriptLoader / jdk.nashorn.internal.runtime.NashornLoader) configured with + // a system class loader. + // + private static final String[] allowedPackagesArray = + { + // following required by jdk.nashorn.internal.objects.Global.initJavaAccess() + "", + "com", + "edu", + "java", + "javax", + "javafx", + "org", + // following required by Nashorn runtime + "java.lang", + "java.lang.invoke", + "java.lang.reflect", + "java.nio.charset", + "java.util", + "java.util.concurrent", + "javax.script", + "sun.reflect", + "jdk.internal.org.objectweb.asm.commons", + "jdk.nashorn.internal.runtime", + "jdk.nashorn.internal.runtime.linker", + // following required by Java Driver + "java.math", + "java.nio", + "java.text", + "com.google.common.base", + "com.google.common.collect", + "com.google.common.reflect", + // following required by UDF + "com.datastax.driver.core", + "com.datastax.driver.core.utils" + }; + + // use a JVM standard ExecutorService as DebuggableThreadPoolExecutor references internal + // classes, which triggers AccessControlException from the UDF sandbox + private static final UDFExecutorService executor = + new UDFExecutorService(new NamedThreadFactory("UserDefinedScriptFunctions", + Thread.MIN_PRIORITY, + udfClassLoader, + new SecurityThreadGroup("UserDefinedScriptFunctions", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList(allowedPackagesArray))), + UDFunction::initializeThread)), + "userscripts"); + ++ private static final ClassFilter classFilter = clsName -> secureResource(clsName.replace('.', '/') + ".class"); ++ ++ private static final NashornScriptEngine scriptEngine; ++ ++ + static + { + ScriptEngineManager scriptEngineManager = new ScriptEngineManager(); - for (ScriptEngineFactory scriptEngineFactory : scriptEngineManager.getEngineFactories()) - { - ScriptEngine scriptEngine = scriptEngineFactory.getScriptEngine(); - boolean compilable = scriptEngine instanceof Compilable; - if (compilable) - { - logger.info("Found scripting engine {} {} - {} {} - language names: {}", - scriptEngineFactory.getEngineName(), scriptEngineFactory.getEngineVersion(), - scriptEngineFactory.getLanguageName(), scriptEngineFactory.getLanguageVersion(), - scriptEngineFactory.getNames()); - for (String name : scriptEngineFactory.getNames()) - scriptEngines.put(name, (Compilable) scriptEngine); - } - } ++ ScriptEngine engine = scriptEngineManager.getEngineByName("nashorn"); ++ NashornScriptEngineFactory factory = engine != null ? (NashornScriptEngineFactory) engine.getFactory() : null; ++ scriptEngine = factory != null ? (NashornScriptEngine) factory.getScriptEngine(new String[]{}, udfClassLoader, classFilter) : null; + + try + { + protectionDomain = new ProtectionDomain(new CodeSource(new URL("udf", "localhost", 0, "/script", new URLStreamHandler() + { + protected URLConnection openConnection(URL u) + { + return null; + } + }), (Certificate[]) null), ThreadAwareSecurityManager.noPermissions); + } + catch (MalformedURLException e) + { + throw new RuntimeException(e); + } + accessControlContext = new AccessControlContext(new ProtectionDomain[]{ protectionDomain }); + } + + private final CompiledScript script; + + ScriptBasedUDFunction(FunctionName name, + List<ColumnIdentifier> argNames, + List<AbstractType<?>> argTypes, + AbstractType<?> returnType, + boolean calledOnNullInput, + String language, + String body) + { + super(name, argNames, argTypes, returnType, calledOnNullInput, language, body); + - Compilable scriptEngine = scriptEngines.get(language); - if (scriptEngine == null) ++ if (!"JavaScript".equalsIgnoreCase(language) || scriptEngine == null) + throw new InvalidRequestException(String.format("Invalid language '%s' for function '%s'", language, name)); + + // execute compilation with no-permissions to prevent evil code e.g. via "static code blocks" / "class initialization" + try + { + this.script = AccessController.doPrivileged((PrivilegedExceptionAction<CompiledScript>) () -> scriptEngine.compile(body), + accessControlContext); + } + catch (PrivilegedActionException x) + { + Throwable e = x.getCause(); + logger.info("Failed to compile function '{}' for language {}: ", name, language, e); + throw new InvalidRequestException( + String.format("Failed to compile function '%s' for language %s: %s", name, language, e)); + } + } + + protected ExecutorService executor() + { + return executor; + } + + public ByteBuffer executeUserDefined(int protocolVersion, List<ByteBuffer> parameters) + { + Object[] params = new Object[argTypes.size()]; + for (int i = 0; i < params.length; i++) + params[i] = compose(protocolVersion, i, parameters.get(i)); + + ScriptContext scriptContext = new SimpleScriptContext(); + scriptContext.setAttribute("javax.script.filename", this.name.toString(), ScriptContext.ENGINE_SCOPE); + Bindings bindings = scriptContext.getBindings(ScriptContext.ENGINE_SCOPE); + for (int i = 0; i < params.length; i++) + bindings.put(argNames.get(i).toString(), params[i]); + + Object result; + try + { + // How to prevent Class.forName() _without_ "help" from the script engine ? + // NOTE: Nashorn enforces a special permission to allow class-loading, which is not granted - so it's fine. + + result = script.eval(scriptContext); + } + catch (ScriptException e) + { + throw new RuntimeException(e); + } + if (result == null) + return null; + + Class<?> javaReturnType = UDHelper.asJavaClass(returnCodec); + Class<?> resultType = result.getClass(); + if (!javaReturnType.isAssignableFrom(resultType)) + { + if (result instanceof Number) + { + Number rNumber = (Number) result; + if (javaReturnType == Integer.class) + result = rNumber.intValue(); + else if (javaReturnType == Long.class) + result = rNumber.longValue(); + else if (javaReturnType == Short.class) + result = rNumber.shortValue(); + else if (javaReturnType == Byte.class) + result = rNumber.byteValue(); + else if (javaReturnType == Float.class) + result = rNumber.floatValue(); + else if (javaReturnType == Double.class) + result = rNumber.doubleValue(); + else if (javaReturnType == BigInteger.class) + { + if (javaReturnType == Integer.class) + result = rNumber.intValue(); + else if (javaReturnType == Short.class) + result = rNumber.shortValue(); + else if (javaReturnType == Byte.class) + result = rNumber.byteValue(); + else if (javaReturnType == Long.class) + result = rNumber.longValue(); + else if (javaReturnType == Float.class) + result = rNumber.floatValue(); + else if (javaReturnType == Double.class) + result = rNumber.doubleValue(); + else if (javaReturnType == BigInteger.class) + { + if (rNumber instanceof BigDecimal) + result = ((BigDecimal) rNumber).toBigInteger(); + else if (rNumber instanceof Double || rNumber instanceof Float) + result = new BigDecimal(rNumber.toString()).toBigInteger(); + else + result = BigInteger.valueOf(rNumber.longValue()); + } + else if (javaReturnType == BigDecimal.class) + // String c'tor of BigDecimal is more accurate than valueOf(double) + result = new BigDecimal(rNumber.toString()); + } + else if (javaReturnType == BigDecimal.class) + // String c'tor of BigDecimal is more accurate than valueOf(double) + result = new BigDecimal(rNumber.toString()); + } + } + + return decompose(protocolVersion, result); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java index af3c894,d3050a5..9c931e8 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java @@@ -382,47 -501,4 +382,46 @@@ public class UFScriptTest extends CQLTe row(1, expected1, expected2)); } } + + @Test + public void testJavascriptDisabled() throws Throwable + { + createTable("CREATE TABLE %s (key int primary key, val double)"); + + DatabaseDescriptor.enableScriptedUserDefinedFunctions(false); + try + { - assertInvalid("double", - "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " + ++ assertInvalid("CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE javascript\n" + + "AS 'Math.sin(val);';"); + } + finally + { + DatabaseDescriptor.enableScriptedUserDefinedFunctions(true); + } + } + + @Test + public void testJavascriptCompileFailure() throws Throwable + { + assertInvalidMessage("Failed to compile function 'cql_test_keyspace.scrinv'", + "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE javascript\n" + + "AS 'foo bar';"); + } + + @Test + public void testScriptInvalidLanguage() throws Throwable + { + assertInvalidMessage("Invalid language 'artificial_intelligence' for function 'cql_test_keyspace.scrinv'", + "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE artificial_intelligence\n" + + "AS 'question for 42?';"); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java index 5c7ca2b,0000000..4e45a8a mode 100644,000000..100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java @@@ -1,258 -1,0 +1,268 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3.validation.entities; + +import java.security.AccessControlException; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.functions.UDHelper; +import org.apache.cassandra.exceptions.FunctionExecutionException; +import org.apache.cassandra.service.ClientWarn; + +public class UFSecurityTest extends CQLTester +{ + @Test + public void testSecurityPermissions() throws Throwable + { + createTable("CREATE TABLE %s (key int primary key, dval double)"); + execute("INSERT INTO %s (key, dval) VALUES (?, ?)", 1, 1d); + + // Java UDFs + + try + { + String fName = createFunction(KEYSPACE_PER_TEST, "double", + "CREATE OR REPLACE FUNCTION %s(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE JAVA\n" + + "AS 'System.getProperty(\"foo.bar.baz\"); return 0d;';"); + execute("SELECT " + fName + "(dval) FROM %s WHERE key=1"); + Assert.fail(); + } + catch (FunctionExecutionException e) + { + assertAccessControlException("System.getProperty(\"foo.bar.baz\"); return 0d;", e); + } + + String[][] typesAndSources = + { + {"", "try { Class.forName(\"" + UDHelper.class.getName() + "\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"}, + {"sun.misc.Unsafe", "sun.misc.Unsafe.getUnsafe(); return 0d;"}, + {"", "try { Class.forName(\"sun.misc.Unsafe\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"}, + {"java.nio.file.FileSystems", "try {" + + " java.nio.file.FileSystems.getDefault(); return 0d;" + + "} catch (Exception t) {" + + " throw new RuntimeException(t);" + + '}'}, + {"java.nio.channels.FileChannel", "try {" + + " java.nio.channels.FileChannel.open(java.nio.file.FileSystems.getDefault().getPath(\"/etc/passwd\")).close(); return 0d;" + + "} catch (Exception t) {" + + " throw new RuntimeException(t);" + + '}'}, + {"java.nio.channels.SocketChannel", "try {" + + " java.nio.channels.SocketChannel.open().close(); return 0d;" + + "} catch (Exception t) {" + + " throw new RuntimeException(t);" + + '}'}, + {"java.io.FileInputStream", "try {" + + " new java.io.FileInputStream(\"./foobar\").close(); return 0d;" + + "} catch (Exception t) {" + + " throw new RuntimeException(t);" + + '}'}, + {"java.lang.Runtime", "try {" + + " java.lang.Runtime.getRuntime(); return 0d;" + + "} catch (Exception t) {" + + " throw new RuntimeException(t);" + + '}'}, + {"org.apache.cassandra.service.StorageService", + "try {" + + " org.apache.cassandra.service.StorageService v = org.apache.cassandra.service.StorageService.instance; v.isShutdown(); return 0d;" + + "} catch (Exception t) {" + + " throw new RuntimeException(t);" + + '}'}, + {"java.net.ServerSocket", "try {" + + " new java.net.ServerSocket().bind(); return 0d;" + + "} catch (Exception t) {" + + " throw new RuntimeException(t);" + + '}'}, + {"java.io.FileOutputStream","try {" + + " new java.io.FileOutputStream(\".foo\"); return 0d;" + + "} catch (Exception t) {" + + " throw new RuntimeException(t);" + + '}'}, + {"java.lang.Runtime", "try {" + + " java.lang.Runtime.getRuntime().exec(\"/tmp/foo\"); return 0d;" + + "} catch (Exception t) {" + + " throw new RuntimeException(t);" + + '}'} + }; + + for (String[] typeAndSource : typesAndSources) + { + assertInvalidMessage(typeAndSource[0] + " cannot be resolved", + "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".invalid_class_access(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE JAVA\n" + + "AS '" + typeAndSource[1] + "';"); + } + + // JavaScript UDFs + + try + { + String fName = createFunction(KEYSPACE_PER_TEST, "double", + "CREATE OR REPLACE FUNCTION %s(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE javascript\n" + + "AS 'org.apache.cassandra.service.StorageService.instance.isShutdown(); 0;';"); + execute("SELECT " + fName + "(dval) FROM %s WHERE key=1"); + Assert.fail("Javascript security check failed"); + } + catch (FunctionExecutionException e) + { + assertAccessControlException("", e); + } + + String[] javascript = + { + "java.lang.management.ManagmentFactory.getThreadMXBean(); 0;", + "new java.io.FileInputStream(\"/tmp/foo\"); 0;", + "new java.io.FileOutputStream(\"/tmp/foo\"); 0;", + "java.nio.file.FileSystems.getDefault().createFileExclusively(\"./foo_bar_baz\"); 0;", + "java.nio.channels.FileChannel.open(java.nio.file.FileSystems.getDefault().getPath(\"/etc/passwd\")); 0;", + "java.nio.channels.SocketChannel.open(); 0;", + "new java.net.ServerSocket().bind(null); 0;", + "var thread = new java.lang.Thread(); thread.start(); 0;", + "java.lang.System.getProperty(\"foo.bar.baz\"); 0;", - "java.lang.Class.forName(\"java.lang.System\"); 0;", + "java.lang.Runtime.getRuntime().exec(\"/tmp/foo\"); 0;", + "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;", + "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;", + // TODO these (ugly) calls are still possible - these can consume CPU (as one could do with an evil loop, too) +// "java.lang.Runtime.getRuntime().traceMethodCalls(true); 0;", +// "java.lang.Runtime.getRuntime().gc(); 0;", +// "java.lang.Runtime.getRuntime(); 0;", + }; + + for (String script : javascript) + { + try + { + String fName = createFunction(KEYSPACE_PER_TEST, "double", + "CREATE OR REPLACE FUNCTION %s(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE javascript\n" + + "AS '" + script + "';"); + execute("SELECT " + fName + "(dval) FROM %s WHERE key=1"); + Assert.fail("Javascript security check failed: " + script); + } + catch (FunctionExecutionException e) + { + assertAccessControlException(script, e); + } + } ++ ++ String script = "java.lang.Class.forName(\"java.lang.System\"); 0;"; ++ String fName = createFunction(KEYSPACE_PER_TEST, "double", ++ "CREATE OR REPLACE FUNCTION %s(val double) " + ++ "RETURNS NULL ON NULL INPUT " + ++ "RETURNS double " + ++ "LANGUAGE javascript\n" + ++ "AS '" + script + "';"); ++ assertInvalidThrowMessage("Java reflection not supported when class filter is present", ++ FunctionExecutionException.class, ++ "SELECT " + fName + "(dval) FROM %s WHERE key=1"); + } + + private static void assertAccessControlException(String script, FunctionExecutionException e) + { + for (Throwable t = e; t != null && t != t.getCause(); t = t.getCause()) + if (t instanceof AccessControlException) + return; + Assert.fail("no AccessControlException for " + script + " (got " + e + ')'); + } + + @Test + public void testAmokUDF() throws Throwable + { + createTable("CREATE TABLE %s (key int primary key, dval double)"); + execute("INSERT INTO %s (key, dval) VALUES (?, ?)", 1, 1d); + + long udfWarnTimeout = DatabaseDescriptor.getUserDefinedFunctionWarnTimeout(); + long udfFailTimeout = DatabaseDescriptor.getUserDefinedFunctionFailTimeout(); + int maxTries = 5; + for (int i = 1; i <= maxTries; i++) + { + try + { + // short timeout + DatabaseDescriptor.setUserDefinedFunctionWarnTimeout(10); + DatabaseDescriptor.setUserDefinedFunctionFailTimeout(250); + // don't kill the unit test... - default policy is "die" + DatabaseDescriptor.setUserFunctionTimeoutPolicy(Config.UserFunctionTimeoutPolicy.ignore); + + ClientWarn.instance.captureWarnings(); + String fName = createFunction(KEYSPACE_PER_TEST, "double", + "CREATE OR REPLACE FUNCTION %s(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE JAVA\n" + + "AS 'long t=System.currentTimeMillis()+110; while (t>System.currentTimeMillis()) { }; return 0d;'"); + execute("SELECT " + fName + "(dval) FROM %s WHERE key=1"); + List<String> warnings = ClientWarn.instance.getWarnings(); + Assert.assertNotNull(warnings); + Assert.assertFalse(warnings.isEmpty()); + ClientWarn.instance.resetWarnings(); + + // Java UDF + + fName = createFunction(KEYSPACE_PER_TEST, "double", + "CREATE OR REPLACE FUNCTION %s(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE JAVA\n" + + "AS 'long t=System.currentTimeMillis()+500; while (t>System.currentTimeMillis()) { }; return 0d;';"); + assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1"); + + // Javascript UDF + + fName = createFunction(KEYSPACE_PER_TEST, "double", + "CREATE OR REPLACE FUNCTION %s(val double) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS double " + + "LANGUAGE JAVASCRIPT\n" + + "AS 'var t=java.lang.System.currentTimeMillis()+500; while (t>java.lang.System.currentTimeMillis()) { }; 0;';"); + assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1"); + + return; + } + catch (Error | RuntimeException e) + { + if (i == maxTries) + throw e; + } + finally + { + // reset to defaults + DatabaseDescriptor.setUserDefinedFunctionWarnTimeout(udfWarnTimeout); + DatabaseDescriptor.setUserDefinedFunctionFailTimeout(udfFailTimeout); + } + } + } + +}
