5 undefined names in Python code

[cclauss]

Undefined names can raise NameError at runtime.

flake8 testing of https://github.com/apache/incubator-mxnet on Python 2.7.14

$ flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics

./docs/mxdoc.py:71:16: F821 undefined name 'root_path'
    pdf_path = root_path + '/docs/api/r/mxnet-r-reference-manual.pdf'
               ^

./example/cnn_chinese_text_classification/data_helpers.py:199:13: F821 undefined name 'word2vec'
    model = word2vec.Word2Vec.load_word2vec_format(path, binary=True)
            ^

./example/cnn_text_classification/data_helpers.py:184:13: F821 undefined name 'word2vec'
    model = word2vec.Word2Vec.load_word2vec_format(path, binary=True)
            ^

./example/fcn-xs/solver.py:66:32: F821 undefined name 'nd'
        self.aux_params = {k : nd.zeros(s) for k, s in zip(aux_names, aux_shapes)}
                               ^

./example/gluon/style_transfer/utils.py:150:59: F821 undefined name 'collections'
        assert isinstance(size, int) or (isinstance(size, collections.Iterable) and len(size) == 2)
                                                          ^

./example/gluon/tree_lstm/dataset.py:85:16: F821 undefined name 'BOS'
        vec = [BOS] if add_bos else []
               ^

./example/gluon/tree_lstm/dataset.py:88:24: F821 undefined name 'EOS'
            vec.append(EOS)
                       ^

./example/model-parallel-lstm/lstm.py:457:8: F821 undefined name 'concat_decode'
    if concat_decode:
       ^

./example/module/lstm_bucketing.py:70:22: F821 undefined name 'DummyIter'
        data_train = DummyIter(data_train)
                     ^

./example/module/lstm_bucketing.py:71:20: F821 undefined name 'DummyIter'
        data_val = DummyIter(data_val)
                   ^

./example/module/train_cifar10.py:181:28: F821 undefined name 'epoch_size'
            step = max(int(epoch_size * args.lr_factor_epoch), 1),
                           ^

./example/neural-style/end_to_end/model_vgg19.py:94:17: F821 undefined name 'out'
    arg_names = out.list_arguments()
                ^

./example/neural-style/end_to_end/model_vgg19.py:105:16: F821 undefined name 'out'
    executor = out.bind(ctx=ctx, args=arg_dict, args_grad=[], grad_req="null")
               ^

./example/profiler/profiler_executor.py:89:15: F821 undefined name 'search_plan'
        sym = search_plan(sym, data=data_shapes)
              ^

./example/profiler/profiler_executor.py:89:37: F821 undefined name 'data_shapes'
        sym = search_plan(sym, data=data_shapes)
                                    ^

./example/reinforcement-learning/a3c/a3c.py:201:16: F821 undefined name 'robo_data'
    dataiter = robo_data.RobosimsDataIter('scenes', args.batch_size, args.input_length, web_viz=True)
               ^

./example/reinforcement-learning/a3c/rl_data.py:134:20: F821 undefined name 'Empty'
            except Empty:
                   ^

./example/rnn/old/gru_bucketing.py:69:22: F821 undefined name 'DummyIter'
        data_train = DummyIter(data_train)
                     ^

./example/rnn/old/gru_bucketing.py:70:20: F821 undefined name 'DummyIter'
        data_val = DummyIter(data_val)
                   ^

./example/rnn/old/lstm_bucketing.py:71:22: F821 undefined name 'DummyIter'
        data_train = DummyIter(data_train)
                     ^

./example/rnn/old/lstm_bucketing.py:72:20: F821 undefined name 'DummyIter'
        data_val = DummyIter(data_val)
                   ^

./example/speech-demo/decode_mxnet.py:157:87: F821 undefined name 'data_train'
                    posteriors = np.log(posteriors[:label[0][0],1:] + 1e-20) - np.log(data_train.label_mean).T
                                                                                      ^

./example/speech-demo/io_func/convert2kaldi.py:71:48: F821 undefined name 'pool_size'
            output_size = int(layers[i + 1]) * pool_size
                                               ^

./example/speech-demo/io_func/utils.py:99:12: F821 undefined name 'T'
    return T.minimum(x * (x > 0), 6)
           ^

./example/speech-demo/io_func/utils.py:107:16: F821 undefined name 'T'
        return T.nnet.sigmoid
               ^

./example/speech-demo/io_func/utils.py:109:16: F821 undefined name 'T'
        return T.tanh
               ^

./example/speech-demo/io_func/utils.py:116:12: F821 undefined name 'T'
    return T.nnet.sigmoid
           ^

./example/speech-demo/io_func/utils.py:119:20: F821 undefined name 'T'
    if act_func == T.nnet.sigmoid:
                   ^

./example/speech-demo/io_func/utils.py:121:20: F821 undefined name 'T'
    if act_func == T.tanh:
                   ^

./example/speech-demo/io_func/utils.py:166:10: F821 undefined name 'T'
    cs = T.add_no_assoc(carry, inc)
         ^

./example/speech-demo/io_func/utils.py:167:9: F821 undefined name 'T'
    s = T.add_no_assoc(total, cs)
        ^

./example/speech-demo/io_func/utils.py:168:20: F821 undefined name 'T'
    update_carry = T.sub(cs, T.sub(s, total))
                   ^

./example/speech-demo/io_func/utils.py:168:30: F821 undefined name 'T'
    update_carry = T.sub(cs, T.sub(s, total))
                             ^

./example/ssd/dataset/pycocotools/coco.py:283:41: F821 undefined name 'm'
                        img = np.ones( (m.shape[0], m.shape[1], 3) )
                                        ^

./example/ssd/dataset/pycocotools/coco.py:283:53: F821 undefined name 'm'
                        img = np.ones( (m.shape[0], m.shape[1], 3) )
                                                    ^

./example/ssd/dataset/pycocotools/coco.py:290:52: F821 undefined name 'm'
                        ax.imshow(np.dstack( (img, m*0.5) ))
                                                   ^

./example/ssd/dataset/pycocotools/coco.py:452:16: F821 undefined name 'm'
        return m
               ^

./example/ssd/symbol/common.py:209:23: F821 undefined name 'start_offset'
         min_sizes = [start_offset] + tmp.tolist()
                      ^

./example/ssd/symbol/common.py:210:46: F821 undefined name 'start_offset'
         max_sizes = tmp.tolist() + [tmp[-1]+start_offset]
                                             ^

./python/mxnet/initializer.py:698:20: F821 undefined name '_INITIALIZER_REGISTRY'
            init = _INITIALIZER_REGISTRY[klass.lower()](**kwargs)
                   ^

./tests/python/gpu/test_operator_gpu.py:41:5: F821 undefined name 'test_support_vector_machine_l1_svm'
del test_support_vector_machine_l1_svm
    ^

./tests/python/gpu/test_operator_gpu.py:42:5: F821 undefined name 'test_support_vector_machine_l2_svm'
del test_support_vector_machine_l2_svm
    ^

./tools/caffe_converter/convert_symbol.py:317:11: F821 undefined name 'mx'
    ret = mx.sym.Group(ret)
          ^

43    F821 undefined name 'root_path'

[szha]

@apache/mxnet-committers: This issue has been inactive for the past 90 days. It has no label and needs triage.

For general "how-to" questions, our user forum (and Chinese version) is a good place to get help.

[ThomasDelteil]

Sorry could you clarify the issue and give a bit of context?

[cclauss]

These are variables that are not defined in the current context which can raise NameError at runtime. Often they are typos or missing imports or code refactoring errors or things like basestring or xrange() being defined in Python 2 but being removed in Python 3.

Just try reading thru the code as if you were the Python interpreter and ask yourself "Where is this variable defined?".

[ThomasDelteil]

Oh I understand now, that's some static code analysis right? Thanks for running that, seems like some files need fixing.

[vandanavk]

Are there any specific advantages that flake8 has over pylint?
pylint has equivalent messages such as 'undefined-variable' (equivalent to F821). So just wondering if it would be easier to add more options to pylintrc and include in the existing build process.

[cclauss]

flake8 will find undefined names that pylint will not because flake8 makes an abstract syntax tree for each Python file and plyint does not. This also explains why flake8 gives different results in Python 2 and Python 3 (the have different ASTs while pylint does not.

Take running code like isinstance(unicode('Hello'), basestring) through pylint on Python 2 and Python 3 and see if there are differences in the output. Now try same with flake8.

[cclauss]

I take it back... You are right.

PyLint is now able to find undefined names.

$ python3 -m pylint *.py

************* Module pylint_test
pylint_test.py:1:0: C0111: Missing module docstring (missing-docstring)
pylint_test.py:1:11: E0602: Undefined variable 'unicode' (undefined-variable)
pylint_test.py:1:29: E0602: Undefined variable 'basestring' (undefined-variable)

[cclauss]

flake8 testing of https://github.com/apache/incubator-mxnet on Python 3.7.0

$ flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics

/home/travis/virtualenv/python3.7.0/lib/python3.7/site-packages/pycodestyle.py:113: FutureWarning: Possible nested set at position 1
  EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]')
./example/reinforcement-learning/a3c/a3c.py:206:16: F821 undefined name 'robo_data'
    dataiter = robo_data.RobosimsDataIter('scenes', args.batch_size, args.input_length, web_viz=True)
               ^
./example/ssd/dataset/pycocotools/coco.py:266:41: F821 undefined name 'm'
                        img = np.ones( (m.shape[0], m.shape[1], 3) )
                                        ^
./example/ssd/dataset/pycocotools/coco.py:266:53: F821 undefined name 'm'
                        img = np.ones( (m.shape[0], m.shape[1], 3) )
                                                    ^
./example/ssd/dataset/pycocotools/coco.py:273:52: F821 undefined name 'm'
                        ax.imshow(np.dstack( (img, m*0.5) ))
                                                   ^
./example/ssd/dataset/pycocotools/coco.py:435:16: F821 undefined name 'm'
        return m
               ^
./example/ssd/symbol/common.py:209:23: F821 undefined name 'start_offset'
         min_sizes = [start_offset] + tmp.tolist()
                      ^
./example/ssd/symbol/common.py:210:46: F821 undefined name 'start_offset'
         max_sizes = tmp.tolist() + [tmp[-1]+start_offset]
                                             ^
./example/fcn-xs/image_segmentaion.py:100:60: F821 undefined name 'ctx'
    fcnxs_args["data"] = mx.nd.array(get_data(args.input), ctx)
                                                           ^
./example/fcn-xs/image_segmentaion.py:103:60: F821 undefined name 'ctx'
    fcnxs_args["softmax_label"] = mx.nd.empty(label_shape, ctx)
                                                           ^
./example/fcn-xs/image_segmentaion.py:104:26: F821 undefined name 'ctx'
    exector = fcnxs.bind(ctx, fcnxs_args, args_grad=None, grad_req="null", aux_states=fcnxs_args)
                         ^
./example/neural-style/end_to_end/model_vgg19.py:94:17: F821 undefined name 'out'
    arg_names = out.list_arguments()
                ^
./example/neural-style/end_to_end/model_vgg19.py:105:16: F821 undefined name 'out'
    executor = out.bind(ctx=ctx, args=arg_dict, args_grad=[], grad_req="null")
               ^
./example/sparse/factorization_machine/metric.py:111:22: F821 undefined name 'label_zero_num'
        total_area = label_zero_num * label_one_num
                     ^
./example/sparse/factorization_machine/metric.py:111:39: F821 undefined name 'label_one_num'
        total_area = label_zero_num * label_one_num
                                      ^
./example/profiler/profiler_executor.py:89:15: F821 undefined name 'search_plan'
        sym = search_plan(sym, data=data_shapes)
              ^
./example/profiler/profiler_executor.py:89:37: F821 undefined name 'data_shapes'
        sym = search_plan(sym, data=data_shapes)
                                    ^
./python/mxnet/initializer.py:700:20: F821 undefined name '_INITIALIZER_REGISTRY'
            init = _INITIALIZER_REGISTRY[klass.lower()](**kwargs)
                   ^
./python/mxnet/optimizer.py:718:34: F821 undefined name 'array'
            weight_master_copy = array(weight, ctx=weight.context, dtype=numpy.float32)
                                 ^
./python/mxnet/optimizer.py:769:16: F821 undefined name 'multiply'
        norm = multiply(v, v).asnumpy().sum()
               ^
./tests/python/unittest/test_engine_import.py:33:13: F821 undefined name 'reload'
            reload(mxnet)
            ^
./tests/nightly/model_backwards_compatibility_check/common.py:214:12: F821 undefined name 'cmp'
    return cmp(normalize(version1), normalize(version2))
           ^
./docs/mxdoc.py:75:16: F821 undefined name 'root_path'
    pdf_path = root_path + '/docs/api/r/mxnet-r-reference-manual.pdf'
               ^
22    F821 undefined name 'root_path'
22

[vandanavk]

@cclauss I'm working on fixing these errors. Will add you to the PRs as I submit them.