Review at https://gerrit.osmocom.org/4441
add osmo-hlr-db-tool, program to migrate from osmo-nitb db
Move macro copy_sqlite3_text_to_buf() to db.h, so it can be used in
hlr_db_tool.c.
Add _dbd_decode_binary() from libdbi to avoid depending on the entire libdbi
just for KI BLOB decoding. Add it in a separate file, copying its own license,
the lGPL.
Offer commandline option --import-nitb-db to read in an old osmo-nitb database
and copy subscriber IMSIs and 2G auth data to OsmoHLR db format.
Anticipate future command line options like --import-csv, so keep the code
generalized.
Change-Id: I0dfa6ec033dd93161c1adc2ce1637195fe5b7a63
---
M src/Makefile.am
M src/db.h
M src/db_hlr.c
A src/dbd_decode_binary.c
A src/hlr_db_tool.c
5 files changed, 473 insertions(+), 11 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-hlr refs/changes/41/4441/1
diff --git a/src/Makefile.am b/src/Makefile.am
index 3b09b7b..9fbb062 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -35,6 +35,7 @@
bin_PROGRAMS = \
osmo-hlr \
+ osmo-hlr-db-tool \
$(NULL)
noinst_PROGRAMS = \
@@ -66,6 +67,21 @@
$(SQLITE3_LIBS) \
$(NULL)
+osmo_hlr_db_tool_SOURCES = \
+ hlr_db_tool.c \
+ db.c \
+ db_hlr.c \
+ logging.c \
+ rand_urandom.c \
+ dbd_decode_binary.c \
+ $(NULL)
+
+osmo_hlr_db_tool_LDADD = \
+ $(LIBOSMOCORE_LIBS) \
+ $(LIBOSMOGSM_LIBS) \
+ $(SQLITE3_LIBS) \
+ $(NULL)
+
db_test_SOURCES = \
auc.c \
db.c \
diff --git a/src/db.h b/src/db.h
index 35e4327..fc8e511 100644
--- a/src/db.h
+++ b/src/db.h
@@ -129,3 +129,14 @@
bool purge_val, bool is_ps);
int hlr_subscr_nam(struct hlr *hlr, struct hlr_subscriber *subscr, bool
nam_val, bool is_ps);
+
+/*! Call sqlite3_column_text() and copy result to a char[].
+ * \param[out] buf A char[] used as sizeof() arg(!) and osmo_strlcpy() target.
+ * \param[in] stmt An sqlite3_stmt*.
+ * \param[in] idx Index in stmt's returned columns.
+ */
+#define copy_sqlite3_text_to_buf(buf, stmt, idx) \
+ do { \
+ const char *_txt = (const char *) sqlite3_column_text(stmt,
idx); \
+ osmo_strlcpy(buf, _txt, sizeof(buf)); \
+ } while (0)
diff --git a/src/db_hlr.c b/src/db_hlr.c
index bae9a5a..c4d4974 100644
--- a/src/db_hlr.c
+++ b/src/db_hlr.c
@@ -35,17 +35,6 @@
#define LOGHLR(imsi, level, fmt, args ...) LOGP(DAUC, level, "IMSI='%s': "
fmt, imsi, ## args)
-/*! Call sqlite3_column_text() and copy result to a char[].
- * \param[out] buf A char[] used as sizeof() arg(!) and osmo_strlcpy() target.
- * \param[in] stmt An sqlite3_stmt*.
- * \param[in] idx Index in stmt's returned columns.
- */
-#define copy_sqlite3_text_to_buf(buf, stmt, idx) \
- do { \
- const char *_txt = (const char *) sqlite3_column_text(stmt,
idx); \
- osmo_strlcpy(buf, _txt, sizeof(buf)); \
- } while (0)
-
/*! Add new subscriber record to the HLR database.
* \param[in,out] dbc database context.
* \param[in] imsi ASCII string of IMSI digits, is validated.
diff --git a/src/dbd_decode_binary.c b/src/dbd_decode_binary.c
new file mode 100644
index 0000000..e1a98ad
--- /dev/null
+++ b/src/dbd_decode_binary.c
@@ -0,0 +1,42 @@
+/* This function is blatantly copied from libdbi, from
+ * https://sourceforge.net/p/libdbi/libdbi/ci/master/tree/src/dbd_helper.c
+ * to save having to depend on the entire libdbi just for KI BLOB decoding.
+ */
+
+/*
+ * libdbi - database independent abstraction layer for C.
+ * Copyright (C) 2001-2003, David Parker and Mark Tobenkin.
+ * http://libdbi.sourceforge.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ * $Id: dbd_helper.c,v 1.44 2011/08/09 11:14:14 mhoenicka Exp $
+ */
+
+#include <sys/types.h>
+
+size_t _dbd_decode_binary(const unsigned char *in, unsigned char *out){
+ int i, e;
+ unsigned char c;
+ e = *(in++);
+ i = 0;
+ while( (c = *(in++))!=0 ){
+ if( c==1 ){
+ c = *(in++) - 1;
+ }
+ out[i++] = c + e;
+ }
+ return (size_t)i;
+}
diff --git a/src/hlr_db_tool.c b/src/hlr_db_tool.c
new file mode 100644
index 0000000..95537b8
--- /dev/null
+++ b/src/hlr_db_tool.c
@@ -0,0 +1,404 @@
+/* (C) 2017 by sysmocom - s.f.m.c. GmbH <[email protected]>
+ *
+ * All Rights Reserved
+ *
+ * Author: Neels Hofmeyr <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <stdlib.h>
+#include <signal.h>
+#include <stdio.h>
+#include <getopt.h>
+#include <inttypes.h>
+#include <string.h>
+
+#include <osmocom/core/logging.h>
+#include <osmocom/core/application.h>
+
+#include "logging.h"
+#include "db.h"
+#include "rand.h"
+
+struct hlr_db_tool_ctx {
+ /* DB context */
+ struct db_context *dbc;
+};
+
+struct hlr_db_tool_ctx *g_hlr_db_tool_ctx;
+
+static struct {
+ const char *db_file;
+ bool bootstrap;
+ const char *import_nitb_db;
+} cmdline_opts = {
+ .db_file = "hlr.db",
+};
+
+static void print_help()
+{
+ printf("Usage: osmo-hlr-db-tool [-l <hlr.db>] [--import-nitb-db
<nitb.db>]\n");
+ printf("Call without arguments to create a new empty ./hlr.db.\n");
+ printf(" -l --database db-name The OsmoHLR database to use,
default '%s'.\n",
+ cmdline_opts.db_file);
+ printf(" -n --import-nitb-db db Add OsmoNITB db's subscribers to
OsmoHLR db.\n");
+ printf(" Be aware that the import is lossy,
only the\n");
+ printf(" IMSI, MSISDN, nam_cs/ps and 2G
auth data are set.\n");
+ printf(" -h --help This text.\n");
+ printf(" -d option --debug=DMAIN:DDB:DAUC Enable debugging.\n");
+ printf(" -s --disable-color Do not print ANSI colors in the
log\n");
+ printf(" -T --timestamp Prefix every log line with a
timestamp.\n");
+ printf(" -e --log-level number Set a global loglevel.\n");
+ printf(" -V --version Print the version of
OsmoHLR-db-tool.\n");
+}
+
+static void print_version(int print_copyright)
+{
+ printf("OsmoHLR-db-tool version %s\n", PACKAGE_VERSION);
+ if (print_copyright)
+ printf("\n"
+ "Copyright (C) 2017 by sysmocom - s.f.m.c. GmbH\n"
+ "License AGPLv3+: GNU AGPL version 3 or later
<http://gnu.org/licenses/agpl-3.0.html>\n"
+ "This is free software: you are free to change and redistribute it.\n"
+ "There is NO WARRANTY, to the extent permitted by law.\n"
+ "\n");
+}
+
+static void handle_options(int argc, char **argv)
+{
+ while (1) {
+ int option_index = 0, c;
+ static struct option long_options[] = {
+ {"help", 0, 0, 'h'},
+ {"database", 1, 0, 'l'},
+ {"import-nitb-db", 1, 0, 'n'},
+ {"debug", 1, 0, 'd'},
+ {"disable-color", 0, 0, 's'},
+ {"timestamp", 0, 0, 'T'},
+ {"log-level", 1, 0, 'e'},
+ {"version", 0, 0, 'V' },
+ {0, 0, 0, 0}
+ };
+
+ c = getopt_long(argc, argv, "hl:n:d:sTe:V",
+ long_options, &option_index);
+ if (c == -1)
+ break;
+
+ switch (c) {
+ case 'h':
+ print_help();
+ exit(0);
+ case 'l':
+ cmdline_opts.db_file = optarg;
+ break;
+ case 'n':
+ cmdline_opts.import_nitb_db = optarg;
+ break;
+ case 'd':
+ log_parse_category_mask(osmo_stderr_target, optarg);
+ break;
+ case 's':
+ log_set_use_color(osmo_stderr_target, 0);
+ break;
+ case 'T':
+ log_set_print_timestamp(osmo_stderr_target, 1);
+ break;
+ case 'e':
+ log_set_log_level(osmo_stderr_target, atoi(optarg));
+ break;
+ case 'V':
+ print_version(1);
+ exit(0);
+ break;
+ default:
+ /* catch unknown options *as well as* missing
arguments. */
+ fprintf(stderr, "Error in command line options.
Exiting.\n");
+ exit(-1);
+ break;
+ }
+ }
+}
+
+static void signal_hdlr(int signal)
+{
+ switch (signal) {
+ case SIGINT:
+ LOGP(DMAIN, LOGL_NOTICE, "Terminating due to SIGINT\n");
+ db_close(g_hlr_db_tool_ctx->dbc);
+ log_fini();
+ talloc_report_full(g_hlr_db_tool_ctx, stderr);
+ exit(0);
+ break;
+ case SIGUSR1:
+ LOGP(DMAIN, LOGL_DEBUG, "Talloc Report due to SIGUSR1\n");
+ talloc_report_full(g_hlr_db_tool_ctx, stderr);
+ break;
+ }
+}
+
+sqlite3 *open_nitb_db(const char *filename)
+{
+ int rc;
+ sqlite3 *nitb_db = NULL;
+
+ rc = sqlite3_open(filename, &nitb_db);
+ if (rc != SQLITE_OK) {
+ LOGP(DDB, LOGL_ERROR, "Unable to open OsmoNITB DB %s; rc =
%d\n", filename, rc);
+ return NULL;
+ }
+
+ return nitb_db;
+}
+
+enum nitb_stmt {
+ NITB_SELECT_SUBSCR,
+ NITB_SELECT_AUTH_KEYS,
+};
+
+static const char *nitb_stmt_sql[] = {
+ [NITB_SELECT_SUBSCR] =
+ "SELECT imsi, id, extension, authorized"
+ " FROM Subscriber"
+ " ORDER BY id",
+ [NITB_SELECT_AUTH_KEYS] =
+ "SELECT algorithm_id, a3a8_ki from authkeys"
+ " WHERE subscriber_id = $subscr_id",
+};
+
+sqlite3_stmt *nitb_stmt[ARRAY_SIZE(nitb_stmt_sql)] = {};
+
+size_t _dbd_decode_binary(const unsigned char *in, unsigned char *out);
+
+void import_nitb_subscr_aud(sqlite3 *nitb_db, const char *imsi, int64_t
nitb_id, int64_t hlr_id)
+{
+ int rc;
+ struct db_context *dbc = g_hlr_db_tool_ctx->dbc;
+ sqlite3_stmt *stmt;
+
+ int count = 0;
+
+ stmt = nitb_stmt[NITB_SELECT_AUTH_KEYS];
+ if (!db_bind_int(stmt, NULL, nitb_id))
+ return;
+
+ while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) {
+ const void *blob;
+ unsigned int blob_size;
+ static unsigned char buf[4096];
+ static char ki[128];
+ int decoded_size;
+ struct sub_auth_data_str aud2g = {
+ .type = OSMO_AUTH_TYPE_GSM,
+ .algo = OSMO_AUTH_ALG_NONE,
+ .u.gsm.ki = ki,
+ };
+
+ aud2g.algo = sqlite3_column_int(stmt, 0);
+
+ if (count) {
+ LOGP(DDB, LOGL_ERROR,
+ "Warning: subscriber has more than one auth key,"
+ " importing only the first key, for IMSI=%s\n",
+ imsi);
+ break;
+ }
+
+ blob = sqlite3_column_blob(stmt, 1);
+ blob_size = sqlite3_column_bytes(stmt, 1);
+
+ if (blob_size > sizeof(buf)) {
+ LOGP(DDB, LOGL_ERROR,
+ "OsmoNITB import to %s: Cannot import auth data
for IMSI %s:"
+ " too large blob: %u\n",
+ dbc->fname, imsi, blob_size);
+ db_remove_reset(stmt);
+ continue;
+ }
+
+ decoded_size = _dbd_decode_binary(blob, buf);
+ osmo_strlcpy(ki, osmo_hexdump_nospc(buf, decoded_size),
sizeof(ki));
+
+ db_subscr_update_aud_by_id(dbc, hlr_id, &aud2g);
+ count ++;
+ }
+
+ if (rc != SQLITE_DONE && rc != SQLITE_ROW) {
+ LOGP(DDB, LOGL_ERROR, "OsmoNITB DB: SQL error: (%d) %s,"
+ " during stmt '%s'",
+ rc, sqlite3_errmsg(nitb_db),
+ nitb_stmt_sql[NITB_SELECT_AUTH_KEYS]);
+ }
+
+ db_remove_reset(stmt);
+}
+
+void import_nitb_subscr(sqlite3 *nitb_db, sqlite3_stmt *stmt)
+{
+ struct db_context *dbc = g_hlr_db_tool_ctx->dbc;
+ int rc;
+ struct hlr_subscriber subscr;
+
+ int64_t nitb_id;
+ int64_t imsi;
+ char imsi_str[32];
+ bool authorized;
+
+ imsi = sqlite3_column_int64(stmt, 0);
+
+ snprintf(imsi_str, sizeof(imsi_str), "%"PRId64, imsi);
+
+ rc = db_subscr_create(dbc, imsi_str);
+ if (rc) {
+ LOGP(DDB, LOGL_ERROR, "OsmoNITB DB import to %s: failed to
create IMSI %s: %d: %s\n",
+ dbc->fname,
+ imsi_str,
+ rc,
+ strerror(rc));
+ /* on error, still attempt to continue */
+ }
+
+ nitb_id = sqlite3_column_int64(stmt, 1);
+ copy_sqlite3_text_to_buf(subscr.msisdn, stmt, 2);
+ authorized = sqlite3_column_int(stmt, 3) ? true : false;
+
+ db_subscr_update_msisdn_by_imsi(dbc, imsi_str, subscr.msisdn);
+ db_subscr_nam(dbc, imsi_str, authorized, true);
+ db_subscr_nam(dbc, imsi_str, authorized, false);
+
+ /* find the just created id */
+ rc = db_subscr_get_by_imsi(dbc, imsi_str, &subscr);
+ if (rc) {
+ LOGP(DDB, LOGL_ERROR, "OsmoNITB DB import to %s: created IMSI
%s,"
+ " but failed to get new subscriber id: %d: %s\n",
+ dbc->fname,
+ imsi_str,
+ rc,
+ strerror(rc));
+ return;
+ }
+
+ OSMO_ASSERT(!strcmp(imsi_str, subscr.imsi));
+
+ import_nitb_subscr_aud(nitb_db, imsi_str, nitb_id, subscr.id);
+}
+
+int import_nitb_db(void)
+{
+ int i;
+ int ret;
+ int rc;
+ const char *sql;
+ sqlite3_stmt *stmt;
+
+ sqlite3 *nitb_db = open_nitb_db(cmdline_opts.import_nitb_db);
+
+ if (!nitb_db)
+ return -1;
+ ret = 0;
+
+ for (i = 0; i < ARRAY_SIZE(nitb_stmt_sql); i++) {
+ sql = nitb_stmt_sql[i];
+ rc = sqlite3_prepare_v2(nitb_db, sql, -1, &nitb_stmt[i], NULL);
+ if (rc != SQLITE_OK) {
+ LOGP(DDB, LOGL_ERROR, "OsmoNITB DB: Unable to prepare
SQL statement '%s'\n", sql);
+ ret = -1;
+ goto out_free;
+ }
+ }
+
+ stmt = nitb_stmt[NITB_SELECT_SUBSCR];
+
+ while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) {
+ import_nitb_subscr(nitb_db, stmt);
+ /* On failure, carry on with the rest. */
+ }
+ if (rc != SQLITE_DONE) {
+ LOGP(DDB, LOGL_ERROR, "OsmoNITB DB: SQL error: (%d) %s,"
+ " during stmt '%s'",
+ rc, sqlite3_errmsg(nitb_db),
+ nitb_stmt_sql[NITB_SELECT_SUBSCR]);
+ goto out_free;
+ }
+
+ db_remove_reset(stmt);
+ sqlite3_finalize(stmt);
+
+out_free:
+ sqlite3_close(nitb_db);
+ return ret;
+}
+
+int main(int argc, char **argv)
+{
+ int rc;
+ int (*main_action)(void);
+ main_action = NULL;
+
+ g_hlr_db_tool_ctx = talloc_zero(NULL, struct hlr_db_tool_ctx);
+ OSMO_ASSERT(g_hlr_db_tool_ctx);
+ talloc_set_name_const(g_hlr_db_tool_ctx, "OsmoHLR-db-tool");
+
+ rc = osmo_init_logging(&hlr_log_info);
+ if (rc < 0) {
+ fprintf(stderr, "Error initializing logging\n");
+ exit(1);
+ }
+
+ handle_options(argc, argv);
+
+ if (cmdline_opts.import_nitb_db) {
+ if (main_action)
+ goto too_many_actions;
+ main_action = import_nitb_db;
+ }
+ /* Future: add more main_actions, besides --import-nitb-db, here. */
+
+ /* Just in case any db actions need randomness */
+ rc = rand_init();
+ if (rc < 0) {
+ LOGP(DMAIN, LOGL_FATAL, "Error initializing random source\n");
+ exit(1);
+ }
+
+ g_hlr_db_tool_ctx->dbc = db_open(g_hlr_db_tool_ctx,
cmdline_opts.db_file);
+ if (!g_hlr_db_tool_ctx->dbc) {
+ LOGP(DMAIN, LOGL_FATAL, "Error opening database\n");
+ exit(1);
+ }
+
+ osmo_init_ignore_signals();
+ signal(SIGINT, &signal_hdlr);
+ signal(SIGUSR1, &signal_hdlr);
+
+ rc = 0;
+ if (main_action)
+ rc = (*main_action)();
+
+ db_close(g_hlr_db_tool_ctx->dbc);
+ log_fini();
+ exit(rc);
+
+too_many_actions:
+ fprintf(stderr, "Too many actions requested.\n");
+ log_fini();
+ exit(1);
+}
+
+/* stubs */
+void lu_op_alloc_conn(void) { OSMO_ASSERT(0); }
+void lu_op_tx_del_subscr_data(void) { OSMO_ASSERT(0); }
+void lu_op_free(void) { OSMO_ASSERT(0); }
--
To view, visit https://gerrit.osmocom.org/4441
To unsubscribe, visit https://gerrit.osmocom.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0dfa6ec033dd93161c1adc2ce1637195fe5b7a63
Gerrit-PatchSet: 1
Gerrit-Project: osmo-hlr
Gerrit-Branch: master
Gerrit-Owner: Neels Hofmeyr <[email protected]>
