This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
The following commit(s) were added to refs/heads/master by this push:
new c265484 Extend checks to all indexes
c265484 is described below
commit c265484bbfd0173aa0a63ea41618da6b5431418a
Author: Sebb <[email protected]>
AuthorDate: Fri Nov 12 23:35:51 2021 +0000
Extend checks to all indexes
---
tools/mappings.py | 59 +++++++++++++++++++++++++++++++++---------------
tools/plugins/elastic.py | 6 +++++
2 files changed, 47 insertions(+), 18 deletions(-)
diff --git a/tools/mappings.py b/tools/mappings.py
index 28e0e69..9d317fe 100755
--- a/tools/mappings.py
+++ b/tools/mappings.py
@@ -15,10 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# utility to check for and add missing mappings. Only applies to mbox index
currently
+# utility to check mappings, report differences and create missing mappings
# ** INITIAL VERSION, liable to change **
+import argparse
import sys
import yaml
from plugins.elastic import Elastic
@@ -29,26 +30,48 @@ if sys.version_info <= (3, 3):
sys.exit(-1)
# the desired mappings
-mapping_file = yaml.safe_load(open("mappings.yaml", "r"))['mbox']['properties']
+mapping_file = yaml.safe_load(open("mappings.yaml", "r"))
elastic = Elastic()
major = elastic.engineMajor()
if major != 7:
print("This script requires ElasticSearch 7 API in order to work!")
sys.exit(-1)
-
-# actual mappings
-mappings =
elastic.indices.get_mapping(index=elastic.db_mbox)[elastic.db_mbox]['mappings']['properties']
-
-if mappings == mapping_file:
- print("Mappings are as expected, hoorah!")
-else:
- unexpected = set(mappings) - set(mapping_file)
- for name in unexpected:
- data = {name: mappings[name]}
- print("Unexpected: " + str(data))
- expected = set(mapping_file) - set(mappings)
- for name in expected:
- data = {name: mapping_file[name]}
- print("Missing: " + str(data))
- elastic.indices.put_mapping(body={'properties': data},
index=elastic.db_mbox)
+
+parser = argparse.ArgumentParser(description="Command line options.")
+parser.add_argument(
+ "--create",
+ dest="create",
+ action="store_true",
+ help="Create the missing mapping(s)",
+)
+args = parser.parse_args()
+
+def check_mapping(index):
+ # expected mappings
+ mappings_expected = mapping_file[index]['properties']
+
+ index_name = elastic.index_name(index)
+ # actual mappings
+ mappings =
elastic.indices.get_mapping(index=index_name)[index_name]['mappings']['properties']
+
+ if mappings == mappings_expected:
+ print("Mappings are as expected, hoorah!")
+ else:
+ unexpected = set(mappings) - set(mappings_expected)
+ for name in unexpected:
+ data = {name: mappings[name]}
+ print("Unexpected: " + str(data))
+ expected = set(mappings_expected) - set(mappings)
+ for name in expected:
+ data = {name: mappings_expected[name]}
+ if args.create:
+ print("Creating the mapping: " + str(data))
+ elastic.indices.put_mapping(body={'properties': data},
index=index_name)
+ else:
+ print("Missing: " + str(data))
+
+
+for type in mapping_file.keys():
+ print("Checking " + type)
+ check_mapping(type)
\ No newline at end of file
diff --git a/tools/plugins/elastic.py b/tools/plugins/elastic.py
index 4681959..c3a5b14 100755
--- a/tools/plugins/elastic.py
+++ b/tools/plugins/elastic.py
@@ -45,6 +45,7 @@ class Elastic:
db_notification: str
db_mailinglist: str
db_auditlog: str
+ dbname: str
def __init__(self):
# Fetch config
@@ -52,6 +53,7 @@ class Elastic:
# Set default names for all indices we use
dbname = config.get('elasticsearch', 'dbname', fallback='ponymail')
+ self.dbname = dbname
self.db_mbox = dbname + '-mbox'
self.db_source = dbname + '-source'
self.db_account = dbname + '-account'
@@ -111,6 +113,10 @@ class Elastic:
# Mimic ES hierarchy: es.indices.xyz()
self.indices = _indices_wrap(self)
+ # convert index type to index name
+ def index_name(self, index):
+ return self.dbname + "-" + index
+
@staticmethod
def libraryVersion():
return ES_VERSION
