marcoabreu commented on a change in pull request #11626: [MXNET-651] MXNet Model Backwards Compatibility Checker URL: https://github.com/apache/incubator-mxnet/pull/11626#discussion_r203428140
########## File path: tests/nightly/model_backwards_compatibility_check/common.py ########## @@ -0,0 +1,210 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import boto3 +import mxnet as mx +import json +import os +import numpy as np +import logging +from mxnet import nd, autograd, gluon +import mxnet.ndarray as nd +from mxnet.gluon.data.vision import transforms, datasets +from mxnet import autograd as ag +import mxnet.ndarray as F +from mxnet.gluon import nn, rnn +import re +import time +import sys +from mxnet.test_utils import assert_almost_equal + +# Set fixed random seeds. +mx.random.seed(7) +np.random.seed(7) +logging.getLogger().setLevel(logging.DEBUG) + +# get the current mxnet version we are running on +mxnet_version = mx.__version__ +model_bucket_name = 'mxnet-model-backwards-compatibility-models' +data_folder = 'mxnet-model-backwards-compatibility-data' +backslash = '/' +s3 = boto3.resource('s3') +ctx = mx.cpu(0) + +def get_module_api_model_definition(): + input = mx.symbol.Variable('data') + input = mx.symbol.Flatten(data=input) + + fc1 = mx.symbol.FullyConnected(data=input, name='fc1', num_hidden=128) + act1 = mx.sym.Activation(data=fc1, name='relu1', act_type="relu") + fc2 = mx.symbol.FullyConnected(data=fc1, name='fc2', num_hidden=2) + op = mx.symbol.SoftmaxOutput(data=fc2, name='softmax') + model = mx.mod.Module(symbol=op, context=ctx, data_names=['data'], label_names=['softmax_label']) + return model + +def save_inference_results(inference_results, model_name): + assert (isinstance(inference_results, mx.ndarray.ndarray.NDArray)) + mx.nd.save(model_name + '-inference', {'inference' : inference_results}) + +def load_inference_results(model_name): + inf_dict = mx.nd.load(model_name+'-inference') + return inf_dict['inference'] + +def save_data_and_labels(test_data, test_labels, model_name): + assert (isinstance(test_data, mx.ndarray.ndarray.NDArray)) + assert (isinstance(test_labels, mx.ndarray.ndarray.NDArray)) + mx.nd.save(model_name + '-data', {'data' : test_data, 'labels' : test_labels}) + +def upload_data_and_labels_to_s3(model_name): + s3 = boto3.client('s3') + file = model_name + '-data' + s3.upload_file(file, model_bucket_name, data_folder + backslash + file) + print ('data files successfully uploaded to s3') + +def upload_model_files_to_s3(files, folder_name): + s3 = boto3.client('s3') + for file in files: + s3.upload_file(file, model_bucket_name, folder_name + file) + +def clean_model_files(files, model_name): + files.append(model_name + '-inference') + files.append(model_name + '-data') + + for file in files: + if os.path.isfile(file): + os.remove(file) + +def download_data_from_s3(model_name): + print ('Downloading data files for %s from bucket %s'%(model_name, model_bucket_name + backslash + data_folder)) + bucket = s3.Bucket(model_bucket_name) + prefix = data_folder + backslash + model_name + '-data' + data_files_meta = list(bucket.objects.filter(Prefix = prefix)) + if len(data_files_meta) == 0: + print ('No data files found for %s' %model_name) Review comment: Please use logging instead of print ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
