From 9d64ff420fdde3c6eaa6d573d41da3b2f587422c Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Fri, 6 Feb 2015 14:38:33 -0500 Subject: [PATCH 01/17] Early commit for issue #1135 Add support migrating single imports from the flask.ext style to flask_ --- scripts/flaskext_migrate.py | 30 ++++++++++++++++++++++++++++++ scripts/temp.py | 19 +++++++++++++++++++ scripts/test.txt | 20 ++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 scripts/flaskext_migrate.py create mode 100644 scripts/temp.py create mode 100644 scripts/test.txt diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py new file mode 100644 index 00000000..6e6367a3 --- /dev/null +++ b/scripts/flaskext_migrate.py @@ -0,0 +1,30 @@ +# from flask.ext import foo => import flask_foo as foo +# from flask.ext.foo import bam => from flask_foo import bam +# import flask.ext.foo => import flask_foo + +import sys + + +def migrate(old_file): + new_file = open("temp.py", "w") + for line in old_file: + if line[0:14] == "from flask.ext": + if line[14] == '.': + import_statement = line[15::].split(' ') + extension = import_statement[0] + line = line.replace("flask.ext." + extension, + "flask_" + extension) + elif line[14] == " ": + import_statement = line[15::].split(' ')[1] + import_statement = import_statement.strip('\n') + line = ("import flask_" + + import_statement + + " as " + + import_statement) + + new_file.write(line) + new_file.close() + +if __name__ == "__main__": + old_file = open(sys.argv[1]) + migrate(old_file) diff --git a/scripts/temp.py b/scripts/temp.py new file mode 100644 index 00000000..38ff98cf --- /dev/null +++ b/scripts/temp.py @@ -0,0 +1,19 @@ +from flask_foo import bam +import flask_foo as foo + +def migrate(old_file): + new_file = open("temp.py", "w") + for line in old_file: + if line[0, 15] is "from flask.ext": + if line[15] == '.': + import_statement = line[16::].split(' ') + extension = import_statement[0] + line = line. replace("flask.ext." + extension, + "flask_" + extension) + else: + pass + + new_file.write(line) + +if __name__ == "__main__": + old_file = open(sys.arv[1]) \ No newline at end of file diff --git a/scripts/test.txt b/scripts/test.txt new file mode 100644 index 00000000..d87c7ad0 --- /dev/null +++ b/scripts/test.txt @@ -0,0 +1,20 @@ +from flask.ext.foo import bam +from flask.ext import foo + + +def migrate(old_file): + new_file = open("temp.py", "w") + for line in old_file: + if line[0, 15] is "from flask.ext": + if line[15] == '.': + import_statement = line[16::].split(' ') + extension = import_statement[0] + line = line. replace("flask.ext." + extension, + "flask_" + extension) + else: + pass + + new_file.write(line) + +if __name__ == "__main__": + old_file = open(sys.arv[1]) \ No newline at end of file From 0b2556f3ec7b3d4be0b7dab9cea8458e07ab18d4 Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Tue, 10 Feb 2015 11:23:16 -0500 Subject: [PATCH 02/17] Add lib2to3 fixer base code Switching to using RedBaron instead of lib2to3, committing to save lib2to3 code in case using Baron is not successful. --- scripts/fix_ext_import.py | 20 ++++++++++++++++++++ scripts/flaskext_migrate.py | 32 ++++++++------------------------ scripts/temp.py | 19 ------------------- scripts/{test.txt => test.py} | 0 4 files changed, 28 insertions(+), 43 deletions(-) create mode 100644 scripts/fix_ext_import.py delete mode 100644 scripts/temp.py rename scripts/{test.txt => test.py} (100%) diff --git a/scripts/fix_ext_import.py b/scripts/fix_ext_import.py new file mode 100644 index 00000000..ca945d8a --- /dev/null +++ b/scripts/fix_ext_import.py @@ -0,0 +1,20 @@ +from lib2to3.fixer_base import BaseFix +from lib2to3.fixer_util import Name, syms + + +class FixExtImport(BaseFix): + + PATTERN = "fixnode='oldname'" + + def transform(self, node, results): + fixnode = results['fixnode'] + fixnode.replace(Name('newname', prefix=fixnode.prefix)) + + if node.type == syms.import_from and \ + getattr(results['imp'], 'value', None) == 'flask.ext': + return 0 + # TODO: Case 2 + + +# CASE 1 - from flask.ext.foo import bam --> from flask_foo import bam +# CASE 2 - from flask.ext import foo --> import flask_foo as foo diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index 6e6367a3..56e28f46 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -1,30 +1,14 @@ -# from flask.ext import foo => import flask_foo as foo -# from flask.ext.foo import bam => from flask_foo import bam -# import flask.ext.foo => import flask_foo +# CASE 1 - from flask.ext.foo import bam --> from flask_foo import bam +# CASE 2 - from flask.ext import foo --> import flask_foo as foo +from redbaron import RedBaron import sys -def migrate(old_file): - new_file = open("temp.py", "w") - for line in old_file: - if line[0:14] == "from flask.ext": - if line[14] == '.': - import_statement = line[15::].split(' ') - extension = import_statement[0] - line = line.replace("flask.ext." + extension, - "flask_" + extension) - elif line[14] == " ": - import_statement = line[15::].split(' ')[1] - import_statement = import_statement.strip('\n') - line = ("import flask_" + - import_statement + - " as " + - import_statement) +with open("test.py", "r") as source_code: + red = RedBaron(source_code.read()) - new_file.write(line) - new_file.close() +print red.dumps() -if __name__ == "__main__": - old_file = open(sys.argv[1]) - migrate(old_file) +# with open("code.py", "w") as source_code: +# source_code.write(red.dumps()) diff --git a/scripts/temp.py b/scripts/temp.py deleted file mode 100644 index 38ff98cf..00000000 --- a/scripts/temp.py +++ /dev/null @@ -1,19 +0,0 @@ -from flask_foo import bam -import flask_foo as foo - -def migrate(old_file): - new_file = open("temp.py", "w") - for line in old_file: - if line[0, 15] is "from flask.ext": - if line[15] == '.': - import_statement = line[16::].split(' ') - extension = import_statement[0] - line = line. replace("flask.ext." + extension, - "flask_" + extension) - else: - pass - - new_file.write(line) - -if __name__ == "__main__": - old_file = open(sys.arv[1]) \ No newline at end of file diff --git a/scripts/test.txt b/scripts/test.py similarity index 100% rename from scripts/test.txt rename to scripts/test.py From 8488a3afa97e12297bdb22586e17b4c0ddf54175 Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Wed, 11 Feb 2015 13:07:24 -0500 Subject: [PATCH 03/17] Change parsing format to use AST via RedBaron Moved away from using manual parsing. Source is parsed using RedBaron to make the FST. Some import formats not yet implemented, and still some bugs to work out. However, the current script works well for the two cases in the file comments. --- scripts/fix_ext_import.py | 20 ------------------ scripts/flaskext_migrate.py | 41 ++++++++++++++++++++++++++++++++----- scripts/test.py | 17 +++++++++++++-- 3 files changed, 51 insertions(+), 27 deletions(-) delete mode 100644 scripts/fix_ext_import.py diff --git a/scripts/fix_ext_import.py b/scripts/fix_ext_import.py deleted file mode 100644 index ca945d8a..00000000 --- a/scripts/fix_ext_import.py +++ /dev/null @@ -1,20 +0,0 @@ -from lib2to3.fixer_base import BaseFix -from lib2to3.fixer_util import Name, syms - - -class FixExtImport(BaseFix): - - PATTERN = "fixnode='oldname'" - - def transform(self, node, results): - fixnode = results['fixnode'] - fixnode.replace(Name('newname', prefix=fixnode.prefix)) - - if node.type == syms.import_from and \ - getattr(results['imp'], 'value', None) == 'flask.ext': - return 0 - # TODO: Case 2 - - -# CASE 1 - from flask.ext.foo import bam --> from flask_foo import bam -# CASE 2 - from flask.ext import foo --> import flask_foo as foo diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index 56e28f46..d3681b7b 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -5,10 +5,41 @@ from redbaron import RedBaron import sys -with open("test.py", "r") as source_code: - red = RedBaron(source_code.read()) +def read_source(input_file): + with open(input_file, "r") as source_code: + red = RedBaron(source_code.read()) + return red -print red.dumps() -# with open("code.py", "w") as source_code: -# source_code.write(red.dumps()) +def write_source(red, input_file): + with open(input_file, "w") as source_code: + source_code.write(red.dumps()) + + +def fix_imports(red): + from_imports = red.find_all("FromImport") + for x in range(len(from_imports)): + values = from_imports[x].value + if (values[0].value == 'flask') and (values[1].value == 'ext'): + # Case 1 + if len(from_imports[x].value) == 3: + package = values[2].value + modules = from_imports[x].modules() + r = "{}," * len(modules) + print modules + from_imports[x].replace("from flask_%s import %s" + % (package, r.format(*modules)[:-1])) + # Case 2 + else: + module = from_imports[x].modules()[0] + from_imports[x].replace("import flask_%s as %s" + % (module, module)) + + return red + + +if __name__ == "__main__": + input_file = sys.argv[1] + ast = read_source(input_file) + new_ast = fix_imports(ast) + write_source(new_ast, input_file) diff --git a/scripts/test.py b/scripts/test.py index d87c7ad0..1c17e0d7 100644 --- a/scripts/test.py +++ b/scripts/test.py @@ -1,6 +1,18 @@ -from flask.ext.foo import bam +from flask.ext.foo import \ + bam, \ + crackle + from flask.ext import foo +from flask.ext.foo import (bam, + a, + b +) + + +from flask.ext import foo + +import sys def migrate(old_file): new_file = open("temp.py", "w") @@ -16,5 +28,6 @@ def migrate(old_file): new_file.write(line) + if __name__ == "__main__": - old_file = open(sys.arv[1]) \ No newline at end of file + old_file = open(sys.arv[1]) From 61d5e6a4d4058accec957901be4d1efe2f3f71fa Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Wed, 11 Feb 2015 13:29:50 -0500 Subject: [PATCH 04/17] Cleanup --- scripts/flaskext_migrate.py | 1 - scripts/test.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index d3681b7b..9b378a09 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -26,7 +26,6 @@ def fix_imports(red): package = values[2].value modules = from_imports[x].modules() r = "{}," * len(modules) - print modules from_imports[x].replace("from flask_%s import %s" % (package, r.format(*modules)[:-1])) # Case 2 diff --git a/scripts/test.py b/scripts/test.py index 1c17e0d7..31b33b58 100644 --- a/scripts/test.py +++ b/scripts/test.py @@ -9,6 +9,7 @@ from flask.ext.foo import (bam, b ) +import flask.ext.foo from flask.ext import foo From c90c4e5e8c796e5dbfa32b30adf68c8582f6162a Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Wed, 11 Feb 2015 14:29:04 -0500 Subject: [PATCH 05/17] Support "import flask.ext.foo" format New commit supports the above import format, however it does not fix function calls elsewhere in the source. Perhaps there should be an error or simply alert the user through stdout(). --- scripts/flaskext_migrate.py | 50 +++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index 9b378a09..53bc80e4 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -1,22 +1,47 @@ -# CASE 1 - from flask.ext.foo import bam --> from flask_foo import bam -# CASE 2 - from flask.ext import foo --> import flask_foo as foo +# Script which modifies source code away from the deprecated "flask.ext" +# format. Does not yet fully support imports in the style: +# +# "import flask.ext.foo" +# +# these are converted to "import flask_foo" in the +# main import statement, but does not handle function calls in the source. +# +# Run in the terminal by typing: `python flaskext_migrate.py ` +# +# Author: Keyan Pishdadian 2015 from redbaron import RedBaron import sys def read_source(input_file): + """Parses the input_file into a RedBaron FST.""" with open(input_file, "r") as source_code: red = RedBaron(source_code.read()) return red def write_source(red, input_file): + """Overwrites the input_file once the FST has been modified.""" with open(input_file, "w") as source_code: source_code.write(red.dumps()) def fix_imports(red): + """Wrapper which fixes "from" style imports and then "import" style.""" + red = fix_standard_imports(red) + red = fix_from_imports(red) + return red + + +def fix_from_imports(red): + """ + Converts "from" style imports to not use "flask.ext". + + Handles: + Case 1: from flask.ext.foo import bam --> from flask_foo import bam + Case 2: from flask.ext import foo --> import flask_foo as foo + """ from_imports = red.find_all("FromImport") for x in range(len(from_imports)): values = from_imports[x].value @@ -33,6 +58,27 @@ def fix_imports(red): module = from_imports[x].modules()[0] from_imports[x].replace("import flask_%s as %s" % (module, module)) + return red + + +def fix_standard_imports(red): + """ + Handles import modification in the form: + import flask.ext.foo" --> import flask_foo + + Does not modify function calls elsewhere in the source outside of the + original import statement. + """ + imports = red.find_all("ImportNode") + for x in range(len(imports)): + values = imports[x].value + try: + if (values[x].value[0].value == 'flask' and + values[x].value[1].value == 'ext'): + package = values[x].value[2].value + imports[x].replace("import flask_%s" % package) + except IndexError: + pass return red From 4a9e34a57476c92a4147f9ecafc357a681f1a19a Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Wed, 11 Feb 2015 14:41:48 -0500 Subject: [PATCH 06/17] Add wrapper for testing fixing functionality --- scripts/flaskext_migrate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index 53bc80e4..b7a1baf0 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -83,8 +83,11 @@ def fix_standard_imports(red): return red -if __name__ == "__main__": - input_file = sys.argv[1] +def fix(input_file): ast = read_source(input_file) new_ast = fix_imports(ast) write_source(new_ast, input_file) + +if __name__ == "__main__": + input_file = sys.argv[1] + fix(input_file) From 96d81c297adddf034c8eef6b2c76cbd9181d6d22 Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Wed, 11 Feb 2015 14:53:39 -0500 Subject: [PATCH 07/17] Change wrapper logic for testing --- scripts/flaskext_migrate.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index b7a1baf0..de7290e4 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -83,10 +83,9 @@ def fix_standard_imports(red): return red -def fix(input_file): - ast = read_source(input_file) - new_ast = fix_imports(ast) - write_source(new_ast, input_file) +def fix(ast): + """Wrapper which allows for testing when not running from shell""" + return fix_imports(ast).dumps() if __name__ == "__main__": input_file = sys.argv[1] From 0cf5881312a2256607cebe36938405a6fe5a820e Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Wed, 11 Feb 2015 15:06:53 -0500 Subject: [PATCH 08/17] Add tests, remove manual testing file #1135 --- scripts/test.py | 34 ----------------------------- tests/test_import_migration.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 34 deletions(-) delete mode 100644 scripts/test.py create mode 100644 tests/test_import_migration.py diff --git a/scripts/test.py b/scripts/test.py deleted file mode 100644 index 31b33b58..00000000 --- a/scripts/test.py +++ /dev/null @@ -1,34 +0,0 @@ -from flask.ext.foo import \ - bam, \ - crackle - -from flask.ext import foo - -from flask.ext.foo import (bam, - a, - b -) - -import flask.ext.foo - -from flask.ext import foo - -import sys - -def migrate(old_file): - new_file = open("temp.py", "w") - for line in old_file: - if line[0, 15] is "from flask.ext": - if line[15] == '.': - import_statement = line[16::].split(' ') - extension = import_statement[0] - line = line. replace("flask.ext." + extension, - "flask_" + extension) - else: - pass - - new_file.write(line) - - -if __name__ == "__main__": - old_file = open(sys.arv[1]) diff --git a/tests/test_import_migration.py b/tests/test_import_migration.py new file mode 100644 index 00000000..dc662aa7 --- /dev/null +++ b/tests/test_import_migration.py @@ -0,0 +1,40 @@ +# Tester for the flaskext_migrate.py module located in flask/scripts/ +# +# Author: Keyan Pishdadian + +import pytest +from redbaron import RedBaron +import flaskext_migrate as migrate + + +def test_simple_from_import(): + red = RedBaron("from flask.ext import foo") + output = migrate.fix(red) + assert output == "import flask_foo as foo" + + +def test_from_to_from_import(): + red = RedBaron("from flask.ext.foo import bar") + output = migrate.fix(red) + assert output == "from flask_foo import bar" + + +def test_multiple_import(): + red = RedBaron("from flask.ext.foo import bar, foobar, something") + output = migrate.fix(red) + assert output == "from flask_foo import bar,foobar,something" + + +def test_multiline_import(): + red = RedBaron("from flask.ext.foo import \ + bar,\ + foobar,\ + something") + output = migrate.fix(red) + assert output == "from flask_foo import bar,foobar,something" + + +def test_module_import(): + red = RedBaron("import flask.ext.foo") + output = migrate.fix(red) + assert output == "import flask_foo" From 37217e85b80c389208cd3396c2cdc4b89bd46bcf Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Wed, 11 Feb 2015 15:22:07 -0500 Subject: [PATCH 09/17] Fix issue with wrapper logic --- scripts/flaskext_migrate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index de7290e4..e23b732b 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -89,4 +89,6 @@ def fix(ast): if __name__ == "__main__": input_file = sys.argv[1] - fix(input_file) + ast = read_source(input_file) + ast = fix_imports(ast) + write_source(ast, input_file) From b759aa2b955cccf7050ece74b84c32089443e47e Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Thu, 12 Feb 2015 11:58:38 -0500 Subject: [PATCH 10/17] Add test for naming module and fix logic to cover --- scripts/flaskext_migrate.py | 39 ++++++++++++++++++++-------------- tests/test_import_migration.py | 11 ++++++++-- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index e23b732b..3d78808f 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -43,21 +43,26 @@ def fix_from_imports(red): Case 2: from flask.ext import foo --> import flask_foo as foo """ from_imports = red.find_all("FromImport") - for x in range(len(from_imports)): - values = from_imports[x].value + for x, node in enumerate(from_imports): + values = node.value if (values[0].value == 'flask') and (values[1].value == 'ext'): # Case 1 - if len(from_imports[x].value) == 3: + if len(node.value) == 3: package = values[2].value - modules = from_imports[x].modules() - r = "{}," * len(modules) - from_imports[x].replace("from flask_%s import %s" - % (package, r.format(*modules)[:-1])) + modules = node.modules() + if len(modules) > 1: + r = "{}," * len(modules) + node.replace("from flask_%s import %s" + % (package, r.format(*modules)[:-1])) + else: + name = node.names()[0] + node.replace("from flask_%s import %s as %s" + % (package, modules.pop(), name)) # Case 2 else: - module = from_imports[x].modules()[0] - from_imports[x].replace("import flask_%s as %s" - % (module, module)) + module = node.modules()[0] + node.replace("import flask_%s as %s" + % (module, module)) return red @@ -70,13 +75,13 @@ def fix_standard_imports(red): original import statement. """ imports = red.find_all("ImportNode") - for x in range(len(imports)): - values = imports[x].value + for x, node in enumerate(imports): try: - if (values[x].value[0].value == 'flask' and - values[x].value[1].value == 'ext'): - package = values[x].value[2].value - imports[x].replace("import flask_%s" % package) + if (node.value[0].value == 'flask' and + node.value[1].value == 'ext'): + package = node.value[2].value + name = node.names()[0] + imports[x].replace("import flask_%s as %s" % (package, name)) except IndexError: pass @@ -88,6 +93,8 @@ def fix(ast): return fix_imports(ast).dumps() if __name__ == "__main__": + if len(sys.argv) < 2: + sys.exit("No filename was included, please try again.") input_file = sys.argv[1] ast = read_source(input_file) ast = fix_imports(ast) diff --git a/tests/test_import_migration.py b/tests/test_import_migration.py index dc662aa7..ddd49142 100644 --- a/tests/test_import_migration.py +++ b/tests/test_import_migration.py @@ -1,7 +1,8 @@ # Tester for the flaskext_migrate.py module located in flask/scripts/ # # Author: Keyan Pishdadian - +import sys +sys.path.append('scripts') import pytest from redbaron import RedBaron import flaskext_migrate as migrate @@ -16,7 +17,7 @@ def test_simple_from_import(): def test_from_to_from_import(): red = RedBaron("from flask.ext.foo import bar") output = migrate.fix(red) - assert output == "from flask_foo import bar" + assert output == "from flask_foo import bar as bar" def test_multiple_import(): @@ -38,3 +39,9 @@ def test_module_import(): red = RedBaron("import flask.ext.foo") output = migrate.fix(red) assert output == "import flask_foo" + + +def test_module_import(): + red = RedBaron("from flask.ext.foo import bar as baz") + output = migrate.fix(red) + assert output == "from flask_foo import bar as baz" From 1479cf80f635164b06767454fdd9e5d37853b307 Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Thu, 12 Feb 2015 13:06:45 -0500 Subject: [PATCH 11/17] Move test file, add RedBaron to tox.ini --- tests/test_import_migration.py | 47 ---------------------------------- tox.ini | 1 + 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 tests/test_import_migration.py diff --git a/tests/test_import_migration.py b/tests/test_import_migration.py deleted file mode 100644 index ddd49142..00000000 --- a/tests/test_import_migration.py +++ /dev/null @@ -1,47 +0,0 @@ -# Tester for the flaskext_migrate.py module located in flask/scripts/ -# -# Author: Keyan Pishdadian -import sys -sys.path.append('scripts') -import pytest -from redbaron import RedBaron -import flaskext_migrate as migrate - - -def test_simple_from_import(): - red = RedBaron("from flask.ext import foo") - output = migrate.fix(red) - assert output == "import flask_foo as foo" - - -def test_from_to_from_import(): - red = RedBaron("from flask.ext.foo import bar") - output = migrate.fix(red) - assert output == "from flask_foo import bar as bar" - - -def test_multiple_import(): - red = RedBaron("from flask.ext.foo import bar, foobar, something") - output = migrate.fix(red) - assert output == "from flask_foo import bar,foobar,something" - - -def test_multiline_import(): - red = RedBaron("from flask.ext.foo import \ - bar,\ - foobar,\ - something") - output = migrate.fix(red) - assert output == "from flask_foo import bar,foobar,something" - - -def test_module_import(): - red = RedBaron("import flask.ext.foo") - output = migrate.fix(red) - assert output == "import flask_foo" - - -def test_module_import(): - red = RedBaron("from flask.ext.foo import bar as baz") - output = migrate.fix(red) - assert output == "from flask_foo import bar as baz" diff --git a/tox.ini b/tox.ini index ba2d2668..3e170d87 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ commands = deps= pytest greenlet + redbaron lowest: Werkzeug==0.7 lowest: Jinja2==2.4 From 9cbe83ef0d03a07a491bd26ecbe508a348ed48a1 Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Thu, 12 Feb 2015 16:41:57 -0500 Subject: [PATCH 12/17] Add a test and cover edge case with parens --- scripts/flaskext_migrate.py | 37 ++++++++++++++++++----- scripts/test_import_migration.py | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 scripts/test_import_migration.py diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index 3d78808f..023508b9 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -50,14 +50,14 @@ def fix_from_imports(red): if len(node.value) == 3: package = values[2].value modules = node.modules() + module_string = _get_modules(modules) if len(modules) > 1: - r = "{}," * len(modules) node.replace("from flask_%s import %s" - % (package, r.format(*modules)[:-1])) + % (package, module_string)) else: name = node.names()[0] node.replace("from flask_%s import %s as %s" - % (package, modules.pop(), name)) + % (package, module_string, name)) # Case 2 else: module = node.modules()[0] @@ -88,13 +88,36 @@ def fix_standard_imports(red): return red -def fix(ast): - """Wrapper which allows for testing when not running from shell""" - return fix_imports(ast).dumps() +def _get_modules(module): + """ + Takes a list of modules and converts into a string -if __name__ == "__main__": + The module list can include parens, this function checks each element in + the list, if there is a paren then it does not add a comma before the next + element. Otherwise a comma and space is added. This is to preserve module + imports which are multi-line and/or occur within parens. While also not + affecting imports which are not enclosed. + """ + modules_string = [cur + ', ' if cur.isalnum() and next.isalnum() + else cur + for (cur, next) in zip(module, module[1:]+[''])] + + return ''.join(modules_string) + + +def check_user_input(): + """Exits and gives error message if no argument is passed in the shell.""" if len(sys.argv) < 2: sys.exit("No filename was included, please try again.") + + +def fix(ast): + """Wrapper which allows for testing when not running from shell.""" + return fix_imports(ast).dumps() + + +if __name__ == "__main__": + check_user_input() input_file = sys.argv[1] ast = read_source(input_file) ast = fix_imports(ast) diff --git a/scripts/test_import_migration.py b/scripts/test_import_migration.py new file mode 100644 index 00000000..71956749 --- /dev/null +++ b/scripts/test_import_migration.py @@ -0,0 +1,51 @@ +# Tester for the flaskext_migrate.py module located in flask/scripts/ +# +# Author: Keyan Pishdadian +import pytest +from redbaron import RedBaron +import flaskext_migrate as migrate + + +def test_simple_from_import(): + red = RedBaron("from flask.ext import foo") + output = migrate.fix(red) + assert output == "import flask_foo as foo" + + +def test_from_to_from_import(): + red = RedBaron("from flask.ext.foo import bar") + output = migrate.fix(red) + assert output == "from flask_foo import bar as bar" + + +def test_multiple_import(): + red = RedBaron("from flask.ext.foo import bar, foobar, something") + output = migrate.fix(red) + assert output == "from flask_foo import bar, foobar, something" + + +def test_multiline_import(): + red = RedBaron("from flask.ext.foo import \ + bar,\ + foobar,\ + something") + output = migrate.fix(red) + assert output == "from flask_foo import bar, foobar, something" + + +def test_module_import(): + red = RedBaron("import flask.ext.foo") + output = migrate.fix(red) + assert output == "import flask_foo" + + +def test_module_import(): + red = RedBaron("from flask.ext.foo import bar as baz") + output = migrate.fix(red) + assert output == "from flask_foo import bar as baz" + + +def test_parens_import(): + red = RedBaron("from flask.ext.foo import (bar, foo, foobar)") + output = migrate.fix(red) + assert output == "from flask_foo import (bar, foo, foobar)" From 4cb311b9455375f6497e7ecb6c931e09b1917d60 Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Fri, 13 Feb 2015 15:27:29 -0500 Subject: [PATCH 13/17] Add support for function call fixing, add tests Addresses #1135, some code cleanup and refactoring. Changes wrapper function which handles testing, further modularized code, added test to cover function call fixing, and fixed duplicate test function name. --- scripts/flaskext_migrate.py | 47 ++++++++++++++++++++++++-------- scripts/test_import_migration.py | 30 ++++++++++++++------ 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index 023508b9..5dcfa664 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -70,18 +70,15 @@ def fix_standard_imports(red): """ Handles import modification in the form: import flask.ext.foo" --> import flask_foo - - Does not modify function calls elsewhere in the source outside of the - original import statement. """ imports = red.find_all("ImportNode") for x, node in enumerate(imports): try: - if (node.value[0].value == 'flask' and - node.value[1].value == 'ext'): - package = node.value[2].value - name = node.names()[0] - imports[x].replace("import flask_%s as %s" % (package, name)) + if (node.value[0].value[0].value == 'flask' and + node.value[0].value[1].value == 'ext'): + package = node.value[0].value[2] + name = node.names()[0].split('.')[-1] + node.replace("import flask_%s as %s" % (package, name)) except IndexError: pass @@ -90,7 +87,7 @@ def fix_standard_imports(red): def _get_modules(module): """ - Takes a list of modules and converts into a string + Takes a list of modules and converts into a string. The module list can include parens, this function checks each element in the list, if there is a paren then it does not add a comma before the next @@ -105,20 +102,46 @@ def _get_modules(module): return ''.join(modules_string) +def fix_function_calls(red): + """ + Modifies function calls in the source to reflect import changes. + + Searches the AST for AtomtrailerNodes and replaces them. + """ + atoms = red.find_all("Atomtrailers") + for x, node in enumerate(atoms): + try: + if (node.value[0].value == 'flask' and + node.value[1].value == 'ext'): + node.replace("flask_foo%s" % node.value[3]) + except IndexError: + pass + + return red + + def check_user_input(): """Exits and gives error message if no argument is passed in the shell.""" if len(sys.argv) < 2: sys.exit("No filename was included, please try again.") -def fix(ast): +def fix_tester(ast): """Wrapper which allows for testing when not running from shell.""" - return fix_imports(ast).dumps() + ast = fix_imports(ast) + ast = fix_function_calls(ast) + return ast.dumps() -if __name__ == "__main__": +def fix(): + """Wrapper for user argument checking and import fixing.""" check_user_input() input_file = sys.argv[1] ast = read_source(input_file) ast = fix_imports(ast) + ast = fix_function_calls(ast) write_source(ast, input_file) + + +if __name__ == "__main__": + fix() diff --git a/scripts/test_import_migration.py b/scripts/test_import_migration.py index 71956749..fa30ef69 100644 --- a/scripts/test_import_migration.py +++ b/scripts/test_import_migration.py @@ -8,19 +8,19 @@ import flaskext_migrate as migrate def test_simple_from_import(): red = RedBaron("from flask.ext import foo") - output = migrate.fix(red) + output = migrate.fix_tester(red) assert output == "import flask_foo as foo" def test_from_to_from_import(): red = RedBaron("from flask.ext.foo import bar") - output = migrate.fix(red) + output = migrate.fix_tester(red) assert output == "from flask_foo import bar as bar" def test_multiple_import(): red = RedBaron("from flask.ext.foo import bar, foobar, something") - output = migrate.fix(red) + output = migrate.fix_tester(red) assert output == "from flask_foo import bar, foobar, something" @@ -29,23 +29,35 @@ def test_multiline_import(): bar,\ foobar,\ something") - output = migrate.fix(red) + output = migrate.fix_tester(red) assert output == "from flask_foo import bar, foobar, something" def test_module_import(): red = RedBaron("import flask.ext.foo") - output = migrate.fix(red) - assert output == "import flask_foo" + output = migrate.fix_tester(red) + assert output == "import flask_foo as foo" -def test_module_import(): +def test_named_module_import(): + red = RedBaron("import flask.ext.foo as foobar") + output = migrate.fix_tester(red) + assert output == "import flask_foo as foobar" + + +def test__named_from_import(): red = RedBaron("from flask.ext.foo import bar as baz") - output = migrate.fix(red) + output = migrate.fix_tester(red) assert output == "from flask_foo import bar as baz" def test_parens_import(): red = RedBaron("from flask.ext.foo import (bar, foo, foobar)") - output = migrate.fix(red) + output = migrate.fix_tester(red) assert output == "from flask_foo import (bar, foo, foobar)" + + +def test_function_call_migration(): + red = RedBaron("flask.ext.foo(var)") + output = migrate.fix_tester(red) + assert output == "flask_foo(var)" From 77ed4661011483cfdb546eb49e89cc8d5cc67be9 Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Fri, 13 Feb 2015 16:02:37 -0500 Subject: [PATCH 14/17] Remove hardcoded 'foo' --- scripts/flaskext_migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index 5dcfa664..0ccea1cb 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -113,7 +113,7 @@ def fix_function_calls(red): try: if (node.value[0].value == 'flask' and node.value[1].value == 'ext'): - node.replace("flask_foo%s" % node.value[3]) + node.replace("flask_%s%s" % (node.value[3], node.value[3])) except IndexError: pass From cd6ec4094730156f27c69b327916e19f6c1c4b2f Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Fri, 13 Feb 2015 18:40:41 -0500 Subject: [PATCH 15/17] Add test and logic for attribute access calls --- scripts/flaskext_migrate.py | 24 +++++++++++++++++++++--- scripts/test_import_migration.py | 10 +++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index 0ccea1cb..a9de0840 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -76,9 +76,12 @@ def fix_standard_imports(red): try: if (node.value[0].value[0].value == 'flask' and node.value[0].value[1].value == 'ext'): - package = node.value[0].value[2] + package = node.value[0].value[2].value name = node.names()[0].split('.')[-1] - node.replace("import flask_%s as %s" % (package, name)) + if name == package: + node.replace("import flask_%s" % (package)) + else: + node.replace("import flask_%s as %s" % (package, name)) except IndexError: pass @@ -113,13 +116,28 @@ def fix_function_calls(red): try: if (node.value[0].value == 'flask' and node.value[1].value == 'ext'): - node.replace("flask_%s%s" % (node.value[3], node.value[3])) + params = _form_function_call(node) + node.replace("flask_%s%s" % (node.value[2], params)) except IndexError: pass return red +def _form_function_call(node): + """ + Reconstructs function call strings when making attribute access calls. + """ + node_vals = node.value + output = "." + for x, param in enumerate(node_vals[3::]): + if param.dumps()[0] == "(": + output = output[0:-1] + param.dumps() + return output + else: + output += param.dumps() + "." + + def check_user_input(): """Exits and gives error message if no argument is passed in the shell.""" if len(sys.argv) < 2: diff --git a/scripts/test_import_migration.py b/scripts/test_import_migration.py index fa30ef69..0220e70a 100644 --- a/scripts/test_import_migration.py +++ b/scripts/test_import_migration.py @@ -36,7 +36,7 @@ def test_multiline_import(): def test_module_import(): red = RedBaron("import flask.ext.foo") output = migrate.fix_tester(red) - assert output == "import flask_foo as foo" + assert output == "import flask_foo" def test_named_module_import(): @@ -61,3 +61,11 @@ def test_function_call_migration(): red = RedBaron("flask.ext.foo(var)") output = migrate.fix_tester(red) assert output == "flask_foo(var)" + + +def test_nested_function_call_migration(): + red = RedBaron("import flask.ext.foo\n\n" + "flask.ext.foo.bar(var)") + output = migrate.fix_tester(red) + assert output == ("import flask_foo\n\n" + "flask_foo.bar(var)") From 5f64971731cfb0b8fbac5783e6ac4615aae2c16d Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Sat, 14 Feb 2015 18:16:28 -0500 Subject: [PATCH 16/17] Update comments to reflect new functionality --- scripts/flaskext_migrate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index a9de0840..575d36e9 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -3,8 +3,6 @@ # # "import flask.ext.foo" # -# these are converted to "import flask_foo" in the -# main import statement, but does not handle function calls in the source. # # Run in the terminal by typing: `python flaskext_migrate.py ` # From 6fa1889afb62fb01e8413584aa7abc293245fcfc Mon Sep 17 00:00:00 2001 From: Keyan Pishdadian Date: Sun, 15 Feb 2015 19:09:08 -0500 Subject: [PATCH 17/17] Update comments to reflect new functionality --- scripts/flaskext_migrate.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/flaskext_migrate.py b/scripts/flaskext_migrate.py index 575d36e9..e24ab95c 100644 --- a/scripts/flaskext_migrate.py +++ b/scripts/flaskext_migrate.py @@ -1,8 +1,5 @@ # Script which modifies source code away from the deprecated "flask.ext" -# format. Does not yet fully support imports in the style: -# -# "import flask.ext.foo" -# +# format. # # Run in the terminal by typing: `python flaskext_migrate.py ` #