Hello community, here is the log from the commit of package python-yara for openSUSE:Factory checked in at 2018-05-29 16:52:10 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-yara (Old) and /work/SRC/openSUSE:Factory/.python-yara.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-yara" Tue May 29 16:52:10 2018 rev:3 rq:612690 version:3.7.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-yara/python-yara.changes 2017-07-21 22:44:59.849983708 +0200 +++ /work/SRC/openSUSE:Factory/.python-yara.new/python-yara.changes 2018-05-29 16:52:13.835916131 +0200 @@ -1,0 +2,11 @@ +Tue May 22 10:38:56 UTC 2018 - [email protected] + +- Update to 3.7.0, tied to changes in yara package itself only +- Run tests + +------------------------------------------------------------------- +Thu Aug 24 13:58:06 UTC 2017 - [email protected] + +- singlespec auto-conversion + +------------------------------------------------------------------- @@ -17,0 +29 @@ + Old: ---- v3.6.1.tar.gz New: ---- v3.7.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-yara.spec ++++++ --- /var/tmp/diff_new_pack.fIjdIQ/_old 2018-05-29 16:52:14.415894792 +0200 +++ /var/tmp/diff_new_pack.fIjdIQ/_new 2018-05-29 16:52:14.415894792 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-yara # -# Copyright (c) 2017 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2018 SUSE LINUX GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -16,38 +16,43 @@ # +%{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-yara -Version: 3.6.1 +Version: 3.7.0 Release: 0 Summary: Python Bindings for YARA (from Virus Total) License: Apache-2.0 Group: Development/Libraries/Python -Url: https://github.com/VirusTotal/yara-python/wiki +URL: https://github.com/VirusTotal/yara-python Source: https://github.com/VirusTotal/yara-python/archive/v%{version}.tar.gz +BuildRequires: %{python_module devel} +BuildRequires: %{python_module setuptools} BuildRequires: fdupes BuildRequires: pkgconfig -BuildRequires: python-setuptools -BuildRequires: pkgconfig(python) -BuildRequires: pkgconfig(yara) = %{version} -BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildRequires: python-rpm-macros +BuildRequires: pkgconfig(yara) >= 3.7 +%python_subpackages %description -python bindings for libyara. YARA is a tool to identify and classify malware samples. +python bindings for libyara. +YARA is a tool to identify and classify malware samples. %prep %setup -q -n yara-python-%{version} %build -python setup.py build --dynamic-linking +%python_build --dynamic-linking %install -python setup.py install --root=%{buildroot} -%fdupes %{buildroot} +%python_install +%python_expand %fdupes %{buildroot}%{$python_sitearch} -%files -%defattr(-,root,root) -%doc LICENSE README.rst -%{python_sitearch}/yara_python-%{version}-py2.7.egg-info -%{python_sitearch}/yara.so +%check +%python_expand PYTHONPATH=%{buildroot}%{$python_sitearch} $python tests.py + +%files %{python_files} +%license LICENSE +%doc README.rst +%{python_sitearch}/* %changelog ++++++ v3.6.1.tar.gz -> v3.7.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yara-python-3.6.1/.gitmodules new/yara-python-3.7.0/.gitmodules --- old/yara-python-3.6.1/.gitmodules 2017-06-06 00:49:35.000000000 +0200 +++ new/yara-python-3.7.0/.gitmodules 2017-11-10 11:14:43.000000000 +0100 @@ -1,3 +1,3 @@ [submodule "yara"] - path = yara +path = yara url=https://[email protected]/VirusTotal/yara.git diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yara-python-3.6.1/.travis.yml new/yara-python-3.7.0/.travis.yml --- old/yara-python-3.6.1/.travis.yml 2017-06-06 00:49:35.000000000 +0200 +++ new/yara-python-3.7.0/.travis.yml 2017-11-10 11:14:43.000000000 +0100 @@ -8,7 +8,12 @@ - "3.5" - "3.6" +before_install: + - sudo apt-get -qq update + - sudo apt-get install -y libmagic-dev + install: + - python setup.py build --enable-magic --enable-dotnet - python setup.py install script: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yara-python-3.6.1/README.rst new/yara-python-3.7.0/README.rst --- old/yara-python-3.6.1/README.rst 2017-06-06 00:49:35.000000000 +0200 +++ new/yara-python-3.7.0/README.rst 2017-11-10 11:14:43.000000000 +0100 @@ -64,4 +64,4 @@ ------------- Find more information about how to use yara-python at -http://yara.readthedocs.org/en/latest/yarapython.html. +https://yara.readthedocs.org/en/latest/yarapython.html. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yara-python-3.6.1/appveyor.yml new/yara-python-3.7.0/appveyor.yml --- old/yara-python-3.6.1/appveyor.yml 2017-06-06 00:49:35.000000000 +0200 +++ new/yara-python-3.7.0/appveyor.yml 2017-11-10 11:14:43.000000000 +0100 @@ -88,6 +88,8 @@ Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` throw "There are newer queued builds for this pull request, failing early." } + - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12" + # Install Python (from the official .msi of http://python.org) and pip when # not already installed. - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 } @@ -110,8 +112,12 @@ # Install the build dependencies of the project. If some dependencies contain # compiled extensions and are not provided as pre-built wheel packages, # pip will build them from source using the MSVC compiler matching the - # target Python version and architecture - - "%CMD_IN_ENV% pip install wheel" + # target Python version and architecture. + + # wheel version 0.29.0 is enforced because version 0.30.0 doesn't support + # Python 2.6 anymore. Once we drop support for Python 2.6 we can use the + # latest version of wheel. + - "%CMD_IN_ENV% pip install wheel==0.29.0" - cd .. - ps: Invoke-WebRequest "https://www.npcglib.org/~stathis/downloads/$env:OPENSSL_LIB.7z"; -OutFile "openssl.7z" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yara-python-3.6.1/setup.py new/yara-python-3.7.0/setup.py --- old/yara-python-3.6.1/setup.py 2017-06-06 00:49:35.000000000 +0200 +++ new/yara-python-3.7.0/setup.py 2017-11-10 11:14:43.000000000 +0100 @@ -159,17 +159,15 @@ for library in self.libraries or []: module.libraries.append(library) - if self.plat_name in ('win32','win-amd64'): - building_for_windows = True - else: - building_for_windows = False + building_for_windows = self.plat_name in ('win32','win-amd64') + building_for_osx = 'macosx' in self.plat_name + building_for_linux = 'linux' in self.plat_name - if 'macosx' in self.plat_name: - building_for_osx = True - else: - building_for_osx = False + if building_for_linux: + module.define_macros.append(('USE_LINUX_PROC', '1')) if building_for_windows: + module.define_macros.append(('USE_WINDOWS_PROC', '1')) module.define_macros.append(('_CRT_SECURE_NO_WARNINGS', '1')) module.libraries.append('kernel32') module.libraries.append('advapi32') @@ -178,6 +176,8 @@ module.libraries.append('ws2_32') if building_for_osx: + module.define_macros.append(('USE_MACH_PROC', '1')) + module.include_dirs.append('/usr/local/opt/openssl/include') module.include_dirs.append('/opt/local/include') module.library_dirs.append('/opt/local/lib') module.include_dirs.append('/usr/local/include') @@ -200,12 +200,14 @@ if (has_function('MD5_Init', libraries=['crypto']) and has_function('SHA256_Init', libraries=['crypto'])): module.define_macros.append(('HASH_MODULE', '1')) + module.define_macros.append(('HAVE_LIBCRYPTO', '1')) module.libraries.append('crypto') else: exclusions.append('yara/libyara/modules/hash.c') if self.enable_magic: module.define_macros.append(('MAGIC_MODULE', '1')) + module.libraries.append('magic') else: exclusions.append('yara/libyara/modules/magic.c') @@ -269,7 +271,7 @@ setup( name='yara-python', - version='3.6.1', + version='3.7.0', description='Python interface for YARA', long_description=readme, license='Apache 2.0', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yara-python-3.6.1/tests.py new/yara-python-3.7.0/tests.py --- old/yara-python-3.6.1/tests.py 2017-06-06 00:49:35.000000000 +0200 +++ new/yara-python-3.7.0/tests.py 2017-11-10 11:14:43.000000000 +0100 @@ -755,6 +755,8 @@ r = yara.compile(p2) self.assertTrue(len(r.match(data='dummy')) == 2) + self.assertRaises(yara.SyntaxError, yara.compile, source='include "test"', includes=False) + def testExternals(self): r = yara.compile(source='rule test { condition: ext_int == 15 }', externals={'ext_int': 15}) @@ -827,6 +829,23 @@ self.assertTrue(rule_data['matches']) self.assertTrue(rule_data['rule'] == 'test') + rule_data = None + + r = yara.compile(source='rule test { condition: false }') + r.match(data='dummy', callback=callback, which_callbacks=yara.CALLBACK_NON_MATCHES) + + self.assertTrue(rule_data['rule'] == 'test') + + def testIncludeCallback(self): + + def callback(requested_filename, filename, namespace): + if requested_filename == 'foo': + return 'rule included {condition: true }' + return None + + r = yara.compile(source='include "foo" rule r { condition: included }', include_callback=callback) + self.assertTrue(r.match(data='dummy')) + def testCompare(self): r = yara.compile(sources={ @@ -939,6 +958,25 @@ self.assertTrue(data['constants']['one'] == 1) self.assertTrue(data['constants']['two'] == 2) + def testRulesIterator(self): + + rules = yara.compile( + source=''' + rule test1 { condition: false } + rule test2 { condition: false } + rule test3 { condition: false } + ''') + + for i, r in enumerate(rules, start=1): + self.assertTrue(r.identifier == 'test%d' % i) + + it = iter(rules) + r = next(it) + self.assertTrue(r.identifier == 'test1') + r = next(it) + self.assertTrue(r.identifier == 'test2') + r = next(it) + self.assertTrue(r.identifier == 'test3') if __name__ == "__main__": diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yara-python-3.6.1/yara-python.c new/yara-python-3.7.0/yara-python.c --- old/yara-python-3.6.1/yara-python.c 2017-06-06 00:49:35.000000000 +0200 +++ new/yara-python-3.7.0/yara-python.c 2017-11-10 11:14:43.000000000 +0100 @@ -68,6 +68,10 @@ For complete documentation please visit:\n\ https://plusvic.github.io/yara\n"; +#if defined(_WIN32) || defined(__CYGWIN__) +#include <string.h> +#define strdup _strdup +#endif // Match object @@ -389,6 +393,7 @@ PyObject* callback; PyObject* modules_data; PyObject* modules_callback; + int which; } CALLBACK_DATA; @@ -424,7 +429,7 @@ case OBJECT_TYPE_STRING: if (object->value.ss != NULL) result = PyBytes_FromStringAndSize( - object->value.ss->c_string, + object->value.ss->c_string, object->value.ss->length); break; @@ -551,6 +556,10 @@ } +#define CALLBACK_MATCHES 0x01 +#define CALLBACK_NON_MATCHES 0x02 +#define CALLBACK_ALL CALLBACK_MATCHES | CALLBACK_NON_MATCHES + int yara_callback( int message, void* message_data, @@ -578,6 +587,7 @@ PyObject* module_data; PyObject* callback_result; PyObject* module_info_dict; + int which = ((CALLBACK_DATA*) user_data)->which; Py_ssize_t data_size; PyGILState_STATE gil_state; @@ -587,7 +597,8 @@ if (message == CALLBACK_MSG_SCAN_FINISHED) return CALLBACK_CONTINUE; - if (message == CALLBACK_MSG_RULE_NOT_MATCHING && callback == NULL) + if (message == CALLBACK_MSG_RULE_NOT_MATCHING && + (callback == NULL || (which & CALLBACK_MATCHES))) return CALLBACK_CONTINUE; if (message == CALLBACK_MSG_IMPORT_MODULE && modules_data == NULL) @@ -758,7 +769,9 @@ } } - if (callback != NULL) + if (callback != NULL && + ((message == CALLBACK_MSG_RULE_MATCHING && (which & CALLBACK_MATCHES)) || + (message == CALLBACK_MSG_RULE_NOT_MATCHING && (which & CALLBACK_NON_MATCHES)))) { Py_INCREF(callback); @@ -903,8 +916,8 @@ { case ERROR_COULD_NOT_ATTACH_TO_PROCESS: return PyErr_Format( - YaraError, - "access denied"); + YaraError, + "access denied"); case ERROR_INSUFFICIENT_MEMORY: return PyErr_NoMemory(); case ERROR_COULD_NOT_OPEN_FILE: @@ -936,6 +949,11 @@ YaraError, "external variable \"%s\" was already defined with a different type", extra); + case ERROR_UNSUPPORTED_FILE_VERSION: + return PyErr_Format( + YaraError, + "rules file \"%s\" is incompatible with this version of YARA", + extra); default: return PyErr_Format( YaraError, @@ -1273,6 +1291,7 @@ if (RULE_IS_NULL(rules->iter_current_rule)) { + rules->iter_current_rule = rules->rules->rules_list_head; PyErr_SetNone(PyExc_StopIteration); return NULL; } @@ -1325,7 +1344,7 @@ static char* kwlist[] = { "filepath", "pid", "data", "externals", "callback", "fast", "timeout", "modules_data", - "modules_callback", NULL + "modules_callback", "which_callbacks", NULL }; char* filepath = NULL; @@ -1348,11 +1367,12 @@ callback_data.callback = NULL; callback_data.modules_data = NULL; callback_data.modules_callback = NULL; + callback_data.which = CALLBACK_ALL; if (PyArg_ParseTupleAndKeywords( args, keywords, - "|sis#OOOiOO", + "|sis#OOOiOOi", kwlist, &filepath, &pid, @@ -1363,7 +1383,8 @@ &fast, &timeout, &callback_data.modules_data, - &callback_data.modules_callback)) + &callback_data.modules_callback, + &callback_data.which)) { if (filepath == NULL && data == NULL && pid == 0) { @@ -1694,6 +1715,103 @@ //////////////////////////////////////////////////////////////////////////////// +const char* yara_include_callback( + const char* include_name, + const char* calling_rule_filename, + const char* calling_rule_namespace, + void* user_data) +{ + PyObject* result; + PyObject* callback = (PyObject*) user_data; + PyObject* py_incl_name = NULL; + PyObject* py_calling_fn = NULL; + PyObject* py_calling_ns = NULL; + PyObject* type = NULL; + PyObject* value = NULL; + PyObject* traceback = NULL; + + const char* cstring_result = NULL; + + PyGILState_STATE gil_state = PyGILState_Ensure(); + + if (include_name != NULL) + { + py_incl_name = PY_STRING(include_name); + } + else //safeguard: should never happen for 'include_name' + { + py_incl_name = Py_None; + Py_INCREF(py_incl_name); + } + + if (calling_rule_filename != NULL) + { + py_calling_fn = PY_STRING(calling_rule_filename); + } + else + { + py_calling_fn = Py_None; + Py_INCREF(py_calling_fn); + } + + if (calling_rule_namespace != NULL) + { + py_calling_ns = PY_STRING(calling_rule_namespace); + } + else + { + py_calling_ns = Py_None; + Py_INCREF(py_calling_ns); + } + + PyErr_Fetch(&type, &value, &traceback); + + result = PyObject_CallFunctionObjArgs( + callback, + py_incl_name, + py_calling_fn, + py_calling_ns, + NULL); + + PyErr_Restore(type, value, traceback); + + Py_DECREF(py_incl_name); + Py_DECREF(py_calling_fn); + Py_DECREF(py_calling_ns); + + if (result != NULL && result != Py_None && PY_STRING_CHECK(result)) + { + //transferring string ownership to C code + cstring_result = strdup(PY_STRING_TO_C(result)); + } + else + { + if (PyErr_Occurred() == NULL) + { + PyErr_Format(PyExc_TypeError, + "'include_callback' function must return a yara rules as an ascii " + "or unicode string"); + } + } + + Py_XDECREF(result); + PyGILState_Release(gil_state); + + return cstring_result; +} + +void yara_include_free( + const char* result_ptr, + void* user_data) +{ + if (result_ptr != NULL) + { + free((void*) result_ptr); + } +} + +//////////////////////////////////////////////////////////////////////////////// + static PyObject* yara_compile( PyObject* self, PyObject* args, @@ -1701,7 +1819,7 @@ { static char *kwlist[] = { "filepath", "source", "file", "filepaths", "sources", - "includes", "externals", "error_on_warning", NULL}; + "includes", "externals", "error_on_warning", "include_callback", NULL}; YR_COMPILER* compiler; YR_RULES* yara_rules; @@ -1718,6 +1836,7 @@ PyObject* includes = NULL; PyObject* externals = NULL; PyObject* error_on_warning = NULL; + PyObject* include_callback = NULL; Py_ssize_t pos = 0; @@ -1731,7 +1850,7 @@ if (PyArg_ParseTupleAndKeywords( args, keywords, - "|ssOOOOOO", + "|ssOOOOOOO", kwlist, &filepath, &source, @@ -1740,8 +1859,10 @@ &sources_dict, &includes, &externals, - &error_on_warning)) + &error_on_warning, + &include_callback)) { + error = yr_compiler_create(&compiler); if (error != ERROR_SUCCESS) @@ -1775,7 +1896,8 @@ if (PyBool_Check(includes)) { // PyObject_IsTrue can return -1 in case of error - compiler->allow_includes = (PyObject_IsTrue(includes) == 1); + if (PyObject_IsTrue(includes) == 1) + yr_compiler_set_include_callback(compiler, NULL, NULL, NULL); } else { @@ -1786,6 +1908,23 @@ } } + if (include_callback != NULL) + { + if (!PyCallable_Check(include_callback)) + { + yr_compiler_destroy(compiler); + return PyErr_Format( + PyExc_TypeError, + "'include_callback' must be callable"); + } + + yr_compiler_set_include_callback( + compiler, + yara_include_callback, + yara_include_free, + include_callback); + } + if (externals != NULL && externals != Py_None) { if (PyDict_Check(externals)) @@ -1805,6 +1944,8 @@ } } + Py_XINCREF(include_callback); + if (filepath != NULL) { fh = fopen(filepath, "r"); @@ -1826,12 +1967,15 @@ else if (file != NULL) { fd = dup(PyObject_AsFileDescriptor(file)); - if (fd != -1) { + + if (fd != -1) + { fh = fdopen(fd, "r"); error = yr_compiler_add_file(compiler, fh, NULL, NULL); fclose(fh); } - else { + else + { result = PyErr_Format( PyExc_TypeError, "'file' is not a file object"); @@ -1954,6 +2098,7 @@ } yr_compiler_destroy(compiler); + Py_XDECREF(include_callback); } return result; @@ -2129,6 +2274,9 @@ PyModule_AddIntConstant(m, "CALLBACK_CONTINUE", 0); PyModule_AddIntConstant(m, "CALLBACK_ABORT", 1); + PyModule_AddIntConstant(m, "CALLBACK_MATCHES", CALLBACK_MATCHES); + PyModule_AddIntConstant(m, "CALLBACK_NON_MATCHES", CALLBACK_NON_MATCHES); + PyModule_AddIntConstant(m, "CALLBACK_ALL", CALLBACK_ALL); PyModule_AddStringConstant(m, "__version__", YR_VERSION); PyModule_AddStringConstant(m, "YARA_VERSION", YR_VERSION); PyModule_AddIntConstant(m, "YARA_VERSION_HEX", YR_VERSION_HEX);
