added apache header, cleaned up some things.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a57f8650 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a57f8650 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a57f8650 Branch: refs/heads/datastax-cass-driver Commit: a57f8650e4573d6ede650b0066821f7fae2849fe Parents: 180bccf Author: Jeff West <[email protected]> Authored: Tue Jul 26 15:52:46 2016 -0700 Committer: Jeff West <[email protected]> Committed: Tue Jul 26 15:52:46 2016 -0700 ---------------------------------------------------------------------- .../es_tools/alias_mover.py | 107 +--- .../es_tools/cluster_shard_allocation.py | 23 +- .../es_tools/command_sender.py | 27 +- .../es_tools/es_index_iterator_reindexer.py | 21 +- .../es_tools/es_searcher.py | 25 +- .../es_tools/index_deleter.py | 22 +- .../es_tools/index_iterator_size_checker.py | 25 +- .../es_tools/index_prefix_checker.py | 21 +- .../es_tools/index_replica_setter.py | 57 ++- .../es_tools/index_shard_allocator.py | 67 ++- .../es_tools/mapping_deleter.py | 19 + .../es_tools/mapping_retriever.py | 19 + .../es_tools/monitor_tasks.py | 19 + .../index_test/document_creator.py | 19 + .../index_test/index_test_mixed_batch.py | 44 +- .../index_test/index_test_single_type_batch.py | 41 +- .../activity_streams/activity_streams.py | 19 + .../samples/beacon-event-example.py | 28 +- .../samples/counter_test.py | 24 +- utils/usergrid-util-python/setup.py | 19 + .../usergrid_tools/general/deleter.py | 19 + .../general/duplicate_name_checker.py | 19 + .../usergrid_tools/general/queue_monitor.py | 19 + .../usergrid_tools/general/url_tester.py | 20 +- .../general/user_confirm_activate.py | 19 + .../usergrid_tools/general/user_creator.py | 19 + .../usergrid_tools/groups/big_group_creater.py | 31 +- .../usergrid_tools/indexing/batch_index_test.py | 19 + .../indexing/entity_index_test.py | 19 + .../usergrid_tools/iterators/simple_iterator.py | 19 + .../iterators/usergrid_cross_region_iterator.py | 26 +- .../iterators/usergrid_iterator.py | 504 +++++++++++++++++++ .../usergrid_tools/library_check.py | 19 + .../migration/usergrid_data_exporter.py | 19 + .../migration/usergrid_data_migrator.py | 23 +- .../parse_importer/parse_importer.py | 19 + .../usergrid_tools/permissions/permissions.py | 19 + .../queue/dlq-iterator-checker.py | 19 + .../usergrid_tools/queue/dlq_requeue.py | 19 + .../usergrid_tools/queue/queue_cleaner.py | 19 + .../usergrid_tools/redis/redis_iterator.py | 19 + .../usergrid_tools/redis/redisscan.py | 19 + 42 files changed, 1355 insertions(+), 218 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/alias_mover.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/alias_mover.py b/utils/usergrid-util-python/es_tools/alias_mover.py index 2a8fe02..3b5caab 100644 --- a/utils/usergrid-util-python/es_tools/alias_mover.py +++ b/utils/usergrid-util-python/es_tools/alias_mover.py @@ -1,94 +1,35 @@ -import json +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ +import json import requests -example_request = { - "actions": [ - { - "remove": { - "index": "apigee-vfmplus", - "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias" - } - }, - { - "remove": { - "index": "apigee-vfmplus", - "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias" - } - }, - { - "remove": { - "index": "apigee-vfmplus", - "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias" - } - }, - { - "remove": { - "index": "apigee-vfmplus", - "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias" - } - }, - { - "remove": { - "index": "apigee-vfmplus", - "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias" - } - }, - { - "remove": { - "index": "apigee-vfmplus", - "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias" - } - }, - { - "add": { - "index": "apigee-vmplus-docvalues", - "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias" - } - }, - { - "add": { - "index": "apigee-vmplus-docvalues", - "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias" - } - }, - { - "add": { - "index": "apigee-vmplus-docvalues", - "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias" - } - }, - { - "add": { - "index": "apigee-vmplus-docvalues", - "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias" - } - }, - { - "add": { - "index": "apigee-vmplus-docvalues", - "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias" - } - }, - { - "add": { - "index": "apigee-vmplus-docvalues", - "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias" - } - } - ] -} - -cluster = 'rug000sr_euwi' +cluster = 'cluster-1' work = { # 'remove': { - # '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'rug000sr_euwi_applications_3', - # '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'rug000sr_euwi_applications_3', + # '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'applications_3', + # '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'applications_3', # }, 'add': { - '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'apigee-vfmplus-1-no-doc-18', - '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'apigee-vfmplus-1-no-doc-18', + '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'my-index-1-no-doc-18', + '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'my-index-1-no-doc-18', } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py index a462124..2e0fcbd 100644 --- a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py +++ b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import time import requests @@ -10,8 +29,8 @@ __author__ = 'Jeff West @ ApigeeCorporation' SHUTDOWN_NODES = True nodes = [ - # 'res206wo', - # 'res207wo', + 'elasticsearch206west', + 'elasticsearch207west', ] base_url = 'http://localhost:9200' http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/command_sender.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/command_sender.py b/utils/usergrid-util-python/es_tools/command_sender.py index 92ecfff..8208e78 100644 --- a/utils/usergrid-util-python/es_tools/command_sender.py +++ b/utils/usergrid-util-python/es_tools/command_sender.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import requests @@ -21,16 +40,16 @@ data = { "move": { "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final", "shard": 14, - "from_node": "res018sy", - "to_node": "res021sy" + "from_node": "elasticsearch018", + "to_node": "elasticsearch021" } }, { "move": { "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final", "shard": 12, - "from_node": "res018sy", - "to_node": "res009sy" + "from_node": "elasticsearch018", + "to_node": "elasticsearch009" } }, http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py index f151fcb..85872e2 100644 --- a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py +++ b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import re from multiprocessing.pool import Pool @@ -5,7 +24,7 @@ import requests # This script iterates an index and issues a PUT request for an empty string to force a reindex of the entity -index_url_template = 'http://res013wo:9200/{index_name}/_search?size={size}&from={from_var}' +index_url_template = 'http://elasticsearch013wo:9200/{index_name}/_search?size={size}&from={from_var}' index_names = [ 'es-index-name' http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/es_searcher.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/es_searcher.py b/utils/usergrid-util-python/es_tools/es_searcher.py index 55e54ef..4c7a297 100644 --- a/utils/usergrid-util-python/es_tools/es_searcher.py +++ b/utils/usergrid-util-python/es_tools/es_searcher.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import requests @@ -5,12 +24,14 @@ import requests __author__ = 'Jeff West @ ApigeeCorporation' -url_template = 'http://localhost:9200/pea000ug_applications_2/_search' +INDEX_NAME='' + +url_template = 'http://localhost:9200/%s/_search' % INDEX_NAME request = { "query": { "term": { - "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,constratus)" + "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,superbad)" } } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_deleter.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/index_deleter.py b/utils/usergrid-util-python/es_tools/index_deleter.py index a697cf8..9b60006 100644 --- a/utils/usergrid-util-python/es_tools/index_deleter.py +++ b/utils/usergrid-util-python/es_tools/index_deleter.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import requests import logging @@ -23,8 +42,7 @@ print 'retrieved %s indices' % len(indices) NUMBER_VALUE = 0 includes = [ - 'rug002sr_euwi', - # 'rug002mr', + 'cluster1', # 'b6768a08-b5d5-11e3-a495-10ddb1de66c3', # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9', ] http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py index 98d4373..03924b2 100644 --- a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py +++ b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import re import traceback @@ -92,7 +111,7 @@ def update_entity_fields(entity, fields): field_value = field.get('long') else: - print 'WTF! %s' % json.dumps(field) + print 'Unexpected field type! %s' % json.dumps(field) return entity_copy entity_copy = update_entity_field(entity_copy, field_name, field_value) @@ -227,10 +246,6 @@ while keep_going: if len(app_id_find) > 0: app_id = app_id_find[0] - if app_id != '5f20f423-f2a8-11e4-a478-12a5923b55dc': - print 'SKIPP APP ID: ' + app_id - continue - entity_id_tmp = source.get('entityId') entity_id_find = re_ent_id.findall(entity_id_tmp) http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_prefix_checker.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/index_prefix_checker.py b/utils/usergrid-util-python/es_tools/index_prefix_checker.py index d72ff3d..0ccb245 100644 --- a/utils/usergrid-util-python/es_tools/index_prefix_checker.py +++ b/utils/usergrid-util-python/es_tools/index_prefix_checker.py @@ -1,7 +1,26 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json from collections import defaultdict import requests -import logging + __author__ = 'Jeff West @ ApigeeCorporation' http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_replica_setter.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/index_replica_setter.py b/utils/usergrid-util-python/es_tools/index_replica_setter.py index 7180fed..1214e48 100644 --- a/utils/usergrid-util-python/es_tools/index_replica_setter.py +++ b/utils/usergrid-util-python/es_tools/index_replica_setter.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + from multiprocessing import Pool import requests import time @@ -32,22 +51,8 @@ payload = { "index.number_of_replicas": NUMBER_VALUE, } -# indices = ['usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_manual'] - includes = [ # '70be096e-c2e1-11e4-8a55-12b4f5e28868', - # 'b0c640af-bc6c-11e4-b078-12b4f5e28868', - # 'e62e465e-bccc-11e4-b078-12b4f5e28868', - # 'd82b6413-bccc-11e4-b078-12b4f5e28868', - # '45914256-c27f-11e4-8a55-12b4f5e28868', - # '2776a776-c27f-11e4-8a55-12b4f5e28868', - # 'a54f878c-bc6c-11e4-b044-0e4cd56e19cd', - # 'ed5b47ea-bccc-11e4-b078-12b4f5e28868', - # 'bd4874ab-bccc-11e4-b044-0e4cd56e19cd', - # '3d748996-c27f-11e4-8a55-12b4f5e28868', - # '1daab807-c27f-11e4-8a55-12b4f5e28868', - # 'd0c4f0da-d961-11e4-849d-12b4f5e28868', - # '93e756ac-bc4e-11e4-92ae-12b4f5e28868', ] excludes = [ @@ -87,18 +92,18 @@ def update_shards(index_name): if update: print index_name - # url = '%s/%s/_settings' % (url_base, index) - # print url - # - # response = requests.get('%s/%s/_settings' % (url_base, index)) - # settings = response.json() - # - # index_settings = settings[index]['settings']['index'] - # - # current_replicas = int(index_settings.get('number_of_replicas')) - # - # if current_replicas == NUMBER_VALUE: - # continue + url = '%s/%s/_settings' % (url_base, index) + print url + + response = requests.get('%s/%s/_settings' % (url_base, index)) + settings = response.json() + + index_settings = settings[index]['settings']['index'] + + current_replicas = int(index_settings.get('number_of_replicas')) + + if current_replicas == NUMBER_VALUE: + return success = False http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_shard_allocator.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/index_shard_allocator.py b/utils/usergrid-util-python/es_tools/index_shard_allocator.py index ecee095..d411744 100644 --- a/utils/usergrid-util-python/es_tools/index_shard_allocator.py +++ b/utils/usergrid-util-python/es_tools/index_shard_allocator.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json from multiprocessing import Pool @@ -11,33 +30,33 @@ __author__ = 'Jeff West @ ApigeeCorporation' nodes_c32xl = [ - 'res000eu', - 'res001eu', - 'res002eu', - 'res003eu', - 'res004eu', - 'res005eu', - 'res009eu', - 'res010eu', - 'res011eu', - 'res012eu', - 'res013eu', - 'res014eu', + 'elasticsearch000eu', + 'elasticsearch001eu', + 'elasticsearch002eu', + 'elasticsearch003eu', + 'elasticsearch004eu', + 'elasticsearch005eu', + 'elasticsearch009eu', + 'elasticsearch010eu', + 'elasticsearch011eu', + 'elasticsearch012eu', + 'elasticsearch013eu', + 'elasticsearch014eu', ] nodes_c34xl = [ - 'res015eu', - 'res018eu', - 'res019eu', - 'res020eu', - 'res021eu', - 'res022eu', - 'res023eu', - 'res024eu', - 'res025eu', - 'res026eu', - 'res027eu', - 'res028eu' + 'elasticsearch015eu', + 'elasticsearch018eu', + 'elasticsearch019eu', + 'elasticsearch020eu', + 'elasticsearch021eu', + 'elasticsearch022eu', + 'elasticsearch023eu', + 'elasticsearch024eu', + 'elasticsearch025eu', + 'elasticsearch026eu', + 'elasticsearch027eu', + 'elasticsearch028eu' ] nodes = nodes_c34xl http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/mapping_deleter.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/mapping_deleter.py b/utils/usergrid-util-python/es_tools/mapping_deleter.py index 74ad898..112cc20 100644 --- a/utils/usergrid-util-python/es_tools/mapping_deleter.py +++ b/utils/usergrid-util-python/es_tools/mapping_deleter.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import requests http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/mapping_retriever.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/mapping_retriever.py b/utils/usergrid-util-python/es_tools/mapping_retriever.py index 0da123b..29fbe11 100644 --- a/utils/usergrid-util-python/es_tools/mapping_retriever.py +++ b/utils/usergrid-util-python/es_tools/mapping_retriever.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import requests http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/monitor_tasks.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/monitor_tasks.py b/utils/usergrid-util-python/es_tools/monitor_tasks.py index df23d49..b444322 100644 --- a/utils/usergrid-util-python/es_tools/monitor_tasks.py +++ b/utils/usergrid-util-python/es_tools/monitor_tasks.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import datetime import requests import time http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/document_creator.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/index_test/document_creator.py b/utils/usergrid-util-python/index_test/document_creator.py index fd544c6..a43e965 100644 --- a/utils/usergrid-util-python/index_test/document_creator.py +++ b/utils/usergrid-util-python/index_test/document_creator.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + from __future__ import print_function from Queue import Empty import json http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/index_test_mixed_batch.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py index d1dd40c..3e6552e 100644 --- a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py +++ b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json from multiprocessing import JoinableQueue, Process import random @@ -6,34 +25,19 @@ import traceback import uuid import time import sys - import argparse import loremipsum import requests from elasticsearch import Elasticsearch - es_hosts = [ - {'host': 'ees000wo', 'port': 9200}, - {'host': 'ees001wo', 'port': 9200}, - {'host': 'ees002wo', 'port': 9200}, - {'host': 'ees003wo', 'port': 9200}, - {'host': 'ees004wo', 'port': 9200}, - {'host': 'ees005wo', 'port': 9200}, - {'host': 'ees006wo', 'port': 9200}, - {'host': 'ees007wo', 'port': 9200}, - {'host': 'ees008wo', 'port': 9200}, - {'host': 'ees009wo', 'port': 9200}, - {'host': 'ees010wo', 'port': 9200}, - {'host': 'ees011wo', 'port': 9200}, - {'host': 'ees012wo', 'port': 9200}, - {'host': 'ees013wo', 'port': 9200}, - {'host': 'ees014wo', 'port': 9200}, - {'host': 'ees015wo', 'port': 9200}, - {'host': 'ees016wo', 'port': 9200}, - {'host': 'ees017wo', 'port': 9200} + {'host': 'elasticsearch000west', 'port': 9200}, + {'host': 'elasticsearch001west', 'port': 9200}, + {'host': 'elasticsearch002west', 'port': 9200}, + {'host': 'elasticsearch003west', 'port': 9200} ] + def parse_args(): parser = argparse.ArgumentParser(description='ElasticSearch Index Test 1') http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/index_test_single_type_batch.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py index e3afdc3..cc25570 100644 --- a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py +++ b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json from multiprocessing import JoinableQueue, Process import random @@ -13,24 +32,10 @@ import requests from elasticsearch import Elasticsearch es_hosts = [ - {'host': 'ees000wo', 'port': 9200}, - {'host': 'ees001wo', 'port': 9200}, - {'host': 'ees002wo', 'port': 9200}, - {'host': 'ees003wo', 'port': 9200}, - {'host': 'ees004wo', 'port': 9200}, - {'host': 'ees005wo', 'port': 9200}, - {'host': 'ees006wo', 'port': 9200}, - {'host': 'ees007wo', 'port': 9200}, - {'host': 'ees008wo', 'port': 9200}, - {'host': 'ees009wo', 'port': 9200}, - {'host': 'ees010wo', 'port': 9200}, - {'host': 'ees011wo', 'port': 9200}, - {'host': 'ees012wo', 'port': 9200}, - {'host': 'ees013wo', 'port': 9200}, - {'host': 'ees014wo', 'port': 9200}, - {'host': 'ees015wo', 'port': 9200}, - {'host': 'ees016wo', 'port': 9200}, - {'host': 'ees017wo', 'port': 9200} + {'host': 'elasticsearch000west', 'port': 9200}, + {'host': 'elasticsearch001west', 'port': 9200}, + {'host': 'elasticsearch002west', 'port': 9200}, + {'host': 'elasticsearch003west', 'port': 9200} ] http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/activity_streams/activity_streams.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py index ce38544..9d95bef 100644 --- a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py +++ b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + # docs page: http://docs.apigee.com/api-baas/content/creating-activity # create user 1 http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/beacon-event-example.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/samples/beacon-event-example.py b/utils/usergrid-util-python/samples/beacon-event-example.py index fc05cdc..8863b35 100644 --- a/utils/usergrid-util-python/samples/beacon-event-example.py +++ b/utils/usergrid-util-python/samples/beacon-event-example.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + # URL Templates for Usergrid import json import random @@ -13,16 +32,11 @@ connection_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb connection_create_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_uuid}" url_data = { - 'api_url': 'https://amer-apibaas-prod.apigee.net/appservices', - 'org': 'jwest-samples', + 'api_url': 'https://usergridhost/basepath', + 'org': 'samples', 'app': 'event-example' } -url_data = { - 'api_url': 'http://usergrid_app.cfapps-01.haas-26.pez.pivotal.io', - 'org': 'jwest', - 'app': 'sandbox' -} session = requests.Session() http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/counter_test.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/samples/counter_test.py b/utils/usergrid-util-python/samples/counter_test.py index 7852b26..4e393b7 100644 --- a/utils/usergrid-util-python/samples/counter_test.py +++ b/utils/usergrid-util-python/samples/counter_test.py @@ -1,4 +1,22 @@ -import datetime +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import time import json @@ -8,7 +26,7 @@ tstamp = time.gmtime() * 1000 s = requests.Session() -s.headers.update({'authorization': 'Bearer YWMt7AHANAKcEeaVR-EahuX8EgAAAVQ7Q56jxQjUsmhJn8rGLTth0XtRrBSIzDA'}) +s.headers.update({'authorization': 'Bearer TOKEN'}) s.headers.update({'content-type': 'application/json'}) url = 'https://host/appservices-new/usergrid/pushtest/events' @@ -26,6 +44,6 @@ print r.status_code time.sleep(30) -r = s.get('https://host/appservices-new/usergrid/pushtest//counters?counter=counters.jeff.west') +r = s.get('https://host/appservices-new/usergrid/pushtest/counters?counter=counters.jeff.west') print r.text http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/setup.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/setup.py b/utils/usergrid-util-python/setup.py index 1f19cb2..83ee6ae 100755 --- a/utils/usergrid-util-python/setup.py +++ b/utils/usergrid-util-python/setup.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + from setuptools import setup, find_packages __author__ = 'Jeff West @ ApigeeCorporation' http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/deleter.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/general/deleter.py b/utils/usergrid-util-python/usergrid_tools/general/deleter.py index 3c53cae..3e21fa4 100644 --- a/utils/usergrid-util-python/usergrid_tools/general/deleter.py +++ b/utils/usergrid-util-python/usergrid_tools/general/deleter.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import traceback import requests http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py index 3682d18..a40e097 100644 --- a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py +++ b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + from usergrid import UsergridQueryIterator ### This iterates a collection using GRAPH and checks whether there are more than on entity with the same name http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py index 37594d1..2ad2950 100644 --- a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py +++ b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import argparse import json import datetime http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/url_tester.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py index 62755df..9d999d6 100644 --- a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py +++ b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py @@ -1,6 +1,24 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import datetime import time - import numpy import requests http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py index 9b1484e..3e94468 100644 --- a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py +++ b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import requests http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/user_creator.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py index d21183d..ace64ee 100644 --- a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py +++ b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import requests http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py index 268b7f8..3e88dfe 100644 --- a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py +++ b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import traceback from multiprocessing import Pool @@ -12,22 +31,14 @@ users = 10000 username_template = 'precisely-10k-%s' url_data = { - "api_url": "https://usergrid-e2e-prod.e2e.apigee.net/appservices-2-1/";, - "org": "tempgrey", + "api_url": "https://usergrid.net";, + "org": "org", "app": "sandbox", "client_id": "", "client_secret": "", } -url_data = { - "api_url": "http://baas-ug002sr.apigee.net/";, - "org": "apigee-vfmplus", - "app": "sandbox", - "client_id": "", - "client_secret": "", - -} collection_url_template = "{api_url}/{org}/{app}/{collection}" add_user_url_template = "{api_url}/{org}/{app}/groups/{group_name}/users/{uuid}" http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py index 6c910dd..1ad8fbc 100644 --- a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py +++ b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py @@ -1,4 +1,23 @@ # -*- coding: utf-8 -*- +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import logging import traceback http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py index 6f193a6..8ec73f6 100644 --- a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py +++ b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py @@ -1,4 +1,23 @@ # -*- coding: utf-8 -*- +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import logging from multiprocessing import Pool http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py index 60cc4a0..c0f2ebd 100644 --- a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py +++ b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import logging import sys import uuid http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py index 35933a2..e5e860f 100644 --- a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py +++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py @@ -1,7 +1,23 @@ -from usergrid import UsergridQuery - -__author__ = 'Jeff West @ ApigeeCorporation' +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ +from usergrid import UsergridQuery from Queue import Empty import argparse import json @@ -12,10 +28,10 @@ from multiprocessing import Process, JoinableQueue import datetime import requests import traceback -from logging.handlers import RotatingFileHandler -import urllib3 import urllib3.contrib.pyopenssl +__author__ = 'Jeff West @ ApigeeCorporation' + urllib3.disable_warnings() urllib3.contrib.pyopenssl.inject_into_urllib3() http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py new file mode 100644 index 0000000..04328ab --- /dev/null +++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py @@ -0,0 +1,504 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + +from Queue import Empty +import json +import logging +import sys +from multiprocessing import Queue, Process +import traceback +from logging.handlers import RotatingFileHandler +import time + +import argparse + +from usergrid import UsergridClient, UsergridError + +__author__ = 'Jeff West @ ApigeeCorporation' + +logger = logging.getLogger('UsergridIterator') + +# SAMPLE CONFIG FILE for source and target +sample_config = { + "endpoint": { + "api_url": "https://api.usergrid.com";, + "limit": 100 + }, + + "credentials": { + "myOrg": { + "client_id": "<<client_id>>", + "client_secret": "<<client_secret>>" + } + } +} + + +def init_logging(file_enabled=False, stdout_enabled=True): + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN) + logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN) + + log_formatter = logging.Formatter(fmt='%(asctime)s | %(name)s | %(processName)s | %(levelname)s | %(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p') + + if file_enabled: + log_file_name = './UsergridIterator.log' + + rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name, + mode='a', + maxBytes=204857600, + backupCount=10) + rotating_file.setFormatter(log_formatter) + rotating_file.setLevel(logging.INFO) + + root_logger.addHandler(rotating_file) + + if stdout_enabled: + stdout_logger = logging.StreamHandler(sys.stdout) + stdout_logger.setFormatter(log_formatter) + stdout_logger.setLevel(logging.INFO) + + root_logger.addHandler(stdout_logger) + + +config = {} + + +class Worker(Process): + """ + The worker is used to perform a set of handler functions in a chain. Work is provided for the Worker thread(s) on + a JoinableQueue. The thread will continue until either 1) it is explicitly terminated or 2) until it does not + receive work on the queue after a consecutive number of attempts (max_empty_count) using the specified timeout + (queue_timeout) + """ + + def __init__(self, + queue, + source_client, + target_client, + max_empty_count=3, + queue_timeout=10, + function_chain=None): + """ + This is an example handler function which can transform an entity. Multiple handler functions can be used to + process a entity. The response is an entity which will get passed to the next handler in the chain + + :param queue: The queue on which to listen for work + :param source_client: The UsergridClient of the source Usergrid instance + :param target_client: The UsergridClient of the target Usergrid instance + :param max_empty_count: The maximum number of times for a worker to not receive work after checking the queue + :param queue_timeout: The timeout for waiting for work on the queue + :param function_chain: An array of function pointers which will be executed in array sequence, expeting the following parameters: org_name, app_name, collection_name, entity, source_client, target_client, attempts=0p + """ + + super(Worker, self).__init__() + logger.warning('Creating worker!') + + if not function_chain: + function_chain = [] + + self.function_chain = function_chain + self.queue = queue + self.source_client = source_client + self.target_client = target_client + self.max_empty_count = max_empty_count + self.queue_timeout = queue_timeout + + def run(self): + logger.info('starting run()...') + keep_going = True + + count_processed = 0 + count_failed = 0 + empty_count = 0 + + while keep_going: + + try: + org, app, collection_name, entity = self.queue.get(timeout=self.queue_timeout) + + empty_count = 0 + success = True + entity_param = entity + + for handler in self.function_chain: + + if entity_param is not None: + try: + entity_param = handler(org, app, collection_name, entity_param, self.source_client, + self.target_client) + except Exception, e: + logger.error(e) + print traceback.format_exc() + success = False + + if success: + count_processed += 1 + logger.info('Processed [%sth] SUCCESS app/collection/name/uuid = %s / %s / %s / %s' % ( + count_processed, app, collection_name, entity.get('name'), entity.get('uuid'))) + else: + count_failed += 1 + logger.warning('Processed [%sth] FAILURE app/collection/name/uuid = %s / %s / %s / %s' % ( + count_processed, app, collection_name, entity.get('name'), entity.get('uuid'))) + + except KeyboardInterrupt, e: + raise e + + except Empty: + logger.warning( + 'No task received after timeout=[%s]! Empty Count=%s' % (self.queue_timeout, empty_count)) + + empty_count += 1 + + if empty_count >= self.max_empty_count: + logger.warning('Stopping work after empty_count=[%s]' % empty_count) + keep_going = False + + logger.info('Worker finished!') + + +def filter_entity(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0): + """ + This is an example handler function which can filter entities. Multiple handler functions can be used to + process a entity. The response is an entity which will get passed to the next handler in the chain + + :param org_name: The org name from whence this entity came + :param app_name: The app name from whence this entity came + :param collection_name: The collection name from whence this entity came + :param entity: The entity retrieved from the source instance + :param source_client: The UsergridClient for the source Usergrid instance + :param target_client: The UsergridClient for the target Usergrid instance + :param attempts: the number of previous attempts this function was run (manual, not part of the framework) + :return: an entity. If response is None then the chain will stop. + """ + + # return None if you want to stop the chain (filter the entity out) + if 'blah' in entity_data: + return None + + # return the entity to keep going + return entity_data + + +def transform_entity(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0): + """ + This is an example handler function which can transform an entity. Multiple handler functions can be used to + process a entity. The response is an entity which will get passed to the next handler in the chain + + :param org_name: The org name from whence this entity came + :param app_name: The app name from whence this entity came + :param collection_name: The collection name from whence this entity came + :param entity: The entity retrieved from the source instance + :param source_client: The UsergridClient for the source Usergrid instance + :param target_client: The UsergridClient for the target Usergrid instance + :param attempts: the number of previous attempts this function was run (manual, not part of the framework) + :return: an entity. If response is None then the chain will stop. + """ + # this just returns the entity with no transform + return entity_data + + +def create_new(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0): + """ + This is an example handler function which can be used to create a new entity in the target instance (based on the + target_client) parameter. Multiple handler functions can be used to process a entity. The response is an entity + which will get passed to the next handler in the chain + + :param org_name: The org name from whence this entity came + :param app_name: The app name from whence this entity came + :param collection_name: The collection name from whence this entity came + :param entity_data: The entity retrieved from the source instance + :param source_client: The UsergridClient for the source Usergrid instance + :param target_client: The UsergridClient for the target Usergrid instance + :param attempts: the number of previous attempts this function was run (manual, not part of the framework) + :return: an entity. If response is None then the chain will stop. + """ + + attempts += 1 + + if 'metadata' in entity_data: entity_data.pop('metadata') + + target_org = config.get('target_org') + target_app = config.get('app_mapping', {}).get(app_name, app_name) + target_collection = config.get('collection_mapping', {}).get(collection_name, collection_name) + + if target_client: + try: + c = target_client.org(target_org).app(target_app).collection(target_collection) + e = c.entity_from_data(entity_data) + e.put() + + except UsergridError, err: + logger.error(err) + raise err + + return None + + +def parse_args(): + parser = argparse.ArgumentParser(description='Usergrid App/Collection Iterator') + + parser.add_argument('-o', '--org', + help='Name of the org to migrate', + type=str, + required=True) + + parser.add_argument('-a', '--app', + help='Multiple, name of apps to include, skip to include all', + default=[], + action='append') + + parser.add_argument('-c', '--collection', + help='Multiple, name of collections to include, skip to include all', + default=[], + action='append') + + parser.add_argument('--ql', + help='The Query string for processing the source collection(s)', + type=str, + default='select *') + + parser.add_argument('-s', '--source_config', + help='The configuration of the source endpoint/org', + type=str, + default='source.json') + + parser.add_argument('-d', '--target_config', + help='The configuration of the target endpoint/org', + type=str, + default='destination.json') + + parser.add_argument('-w', '--workers', + help='The number of worker threads', + type=int, + default=1) + + parser.add_argument('-f', '--force', + help='Force an update regardless of modified date', + type=bool, + default=False) + + parser.add_argument('--max_empty_count', + help='The number of iterations for an individual worker to receive no work before stopping', + type=int, + default=3) + + parser.add_argument('--queue_timeout', + help='The duration in seconds for an individual worker queue poll before Empty is raised', + type=int, + default=10) + + parser.add_argument('--map_app', + help="A colon-separated string such as 'apples:oranges' which indicates to put data from the app named 'apples' from the source endpoint into app named 'oranges' in the target endpoint", + default=[], + action='append') + + parser.add_argument('--map_collection', + help="A colon-separated string such as 'cats:dogs' which indicates to put data from collections named 'cats' from the source endpoint into a collection named 'dogs' in the target endpoint, applicable to all apps", + default=[], + action='append') + + parser.add_argument('--target_org', + help="The org name at the Usergrid destination instance", + type=str) + + my_args = parser.parse_args(sys.argv[1:]) + + return vars(my_args) + + +def init(): + global config + + config['collection_mapping'] = {} + config['app_mapping'] = {} + config['org_mapping'] = {} + + with open(config.get('source_config'), 'r') as f: + config['source_config'] = json.load(f) + + with open(config.get('target_config'), 'r') as f: + config['target_config'] = json.load(f) + + for mapping in config.get('map_collection', []): + parts = mapping.split(':') + + if len(parts) == 2: + config['collection_mapping'][parts[0]] = parts[1] + else: + logger.warning('Skipping malformed Collection mapping: [%s]' % mapping) + + for mapping in config.get('map_app', []): + parts = mapping.split(':') + + if len(parts) == 2: + config['app_mapping'][parts[0]] = parts[1] + else: + logger.warning('Skipping malformed App mapping: [%s]' % mapping) + + for mapping in config.get('map_org', []): + parts = mapping.split(':') + + if len(parts) == 2: + config['org_mapping'][parts[0]] = parts[1] + else: + logger.warning('Skipping Org mapping: [%s]' % mapping) + + if 'source_config' in config: + config['source_endpoint'] = config['source_config'].get('endpoint').copy() + config['source_endpoint'].update(config['source_config']['credentials'][config['org']]) + + config['target_org'] = config['target_org'] if config['target_org'] else config['org'] + + if 'target_config' in config: + config['target_endpoint'] = config['target_config'].get('endpoint').copy() + config['target_endpoint'].update(config['target_config']['credentials'][config['target_org']]) + + +def wait_for(arr_threads, sleep_time=3): + """ + This function pauses the thread until the array of threads which is provided all stop working + + :param arr_threads: an array of Process objects to monitor + :param sleep_time: the time to sleep between evaluating the array for completion + :return: None + """ + threads_working = 100 + + while threads_working > 0: + threads_working = 0 + + for t in arr_threads: + + if t.is_alive(): + threads_working += 1 + + if threads_working > 0: + logger.warn('Waiting for [%s] threads to finish...' % threads_working) + time.sleep(sleep_time) + + logger.warn('Worker Threads finished!') + + +class UsergridIterator: + def __init__(self): + pass + + def get_to_work(self): + global config + + queue = Queue() + logger.warning('Starting workers...') + + apps_to_process = config.get('app') + collections_to_process = config.get('collection') + source_org = config['org'] + target_org = config.get('target_org', config.get('org')) + + source_client = None + target_client = None + + try: + source_client = UsergridClient(api_url=config['source_endpoint']['api_url'], + org_name=source_org) + source_client.authenticate_management_client( + client_credentials=config['source_config']['credentials'][source_org]) + + except UsergridError, e: + logger.critical(e) + exit() + + if 'target_endpoint' in config: + try: + target_client = UsergridClient(api_url=config['target_endpoint']['api_url'], + org_name=target_org) + target_client.authenticate_management_client( + client_credentials=config['target_config']['credentials'][target_org]) + + except UsergridError, e: + logger.critical(e) + exit() + + function_chain = [filter_entity, transform_entity, create_new] + + workers = [Worker(queue=queue, + source_client=source_client, + target_client=target_client, + function_chain=function_chain, + max_empty_count=config.get('max_empty_count', 3), + queue_timeout=config.get('queue_timeout', 10)) + + for x in xrange(config.get('workers', 1))] + + [w.start() for w in workers] + + for app in source_client.list_apps(): + + if len(apps_to_process) > 0 and app not in apps_to_process: + logger.warning('Skipping app=[%s]' % app) + continue + + logger.warning('Processing app=[%s]' % app) + + source_app = source_client.organization(source_org).application(app) + + for collection_name, collection in source_app.list_collections().iteritems(): + + if collection_name in ['events', 'queues']: + logger.warning('Skipping internal collection=[%s]' % collection_name) + continue + + if len(collections_to_process) > 0 and collection_name not in collections_to_process: + logger.warning('Skipping collection=[%s]' % collection_name) + continue + + logger.warning('Processing collection=%s' % collection_name) + + counter = 0 + + try: + for entity in collection.query(ql=config.get('ql'), + limit=config.get('source_endpoint', {}).get('limit', 100)): + counter += 1 + queue.put((config.get('org'), app, collection_name, entity)) + + except KeyboardInterrupt: + [w.terminate() for w in workers] + + logger.info('Publishing entities complete!') + + wait_for(workers) + + logger.info('All done!!') + + +def main(): + global config + config = parse_args() + init() + + init_logging() + + UsergridIterator().get_to_work() + + +if __name__ == '__main__': + main() http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/library_check.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/library_check.py b/utils/usergrid-util-python/usergrid_tools/library_check.py index f326987..0fc6b2f 100644 --- a/utils/usergrid-util-python/usergrid_tools/library_check.py +++ b/utils/usergrid-util-python/usergrid_tools/library_check.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import traceback url_data = { http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py index 19d6f24..e374fbc 100644 --- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py +++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import os import uuid from Queue import Empty http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py index 30ecc26..c99aa12 100644 --- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py +++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import os import uuid from Queue import Empty @@ -7,9 +26,7 @@ import logging import sys from multiprocessing import Queue, Process from sets import Set - import time_uuid - import datetime from cloghandler import ConcurrentRotatingFileHandler import requests @@ -17,9 +34,7 @@ import traceback import redis import time from sys import platform as _platform - import signal - from requests.auth import HTTPBasicAuth from usergrid import UsergridQueryIterator import urllib3 http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py index 3a2d864..0989401 100644 --- a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py +++ b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json import logging from logging.handlers import RotatingFileHandler http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py index e859843..95730d5 100644 --- a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py +++ b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py @@ -1,3 +1,22 @@ +# */ +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, +# * software distributed under the License is distributed on an +# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# * KIND, either express or implied. See the License for the +# * specific language governing permissions and limitations +# * under the License. +# */ + import json from multiprocessing import Pool
