Merge pull request #23 from typecode/develop

Didericis · web-flow · commit 7be78e40e7a9 · 2018-12-02T14:07:39.000-05:00
Version 0.3.2
diff --git a/create_readme.py b/create_readme.py
@@ -4,7 +4,7 @@
 import os
 import chevron
 
-PLUGINS_PATH = os.path.join(os.path.dirname(__file__), 'sykle.plugins')
+PLUGINS_PATH = os.path.join(os.path.dirname(__file__), 'sykle/plugins')
 TEMPLATE_PATH = os.path.join(os.path.dirname(__file__), 'README.mustache')
 README_PATH = os.path.join(os.path.dirname(__file__), 'README.md')
 
@@ -22,6 +22,7 @@ def create_readme(template, destination, docstring):
 
     if os.path.isfile(destination):
         os.chmod(destination, 0o644)
+
     with open(destination, 'w+', encoding='utf-8') as f:
         f.write(long_description)
 
diff --git a/sykle/__init__.py b/sykle/__init__.py
@@ -1,6 +1,6 @@
 # __init__.py
 
-__version__ = '0.3.1'
+__version__ = '0.3.2'
 
 
 class Sykle():
diff --git a/sykle/plugins/sync_pg_data/README.md b/sykle/plugins/sync_pg_data/README.md
@@ -2,13 +2,19 @@
 
 Wrapper around `pg_restore`, `pg_dump` and `psql` that allows you to sync data from one location to another.
 
-### Data only
+### Usage Instructions
 
-This command does NOT do a full database dump and restore. Dumps and restores DATA ONLY so we can sync apps that still have active connections. This means all tables in the source that you would like to link to the destination must be present.
+This command will sync BOTH data and the schema. This has a couple implications:
 
-### Truncation
-
-In order to avoid foreign key constraint errors when restoring data, any existing data must be removed. This means *THE ORIGINAL DESTINATION DATA WILL BE REMOVED*.
+- You will need to ensure that the place you are syncing FROM has code that is either the SAME or is OLDER than the place you are syncing to. Although you **COULD** dump data from a newer codebase into data from an older codebase, if the database schema has changed, the code and migrations will not work properly.
+  - `production -> staging` ✅
+  - `staging -> development` ✅
+  - `production -> development` ✅
+  - ~~`staging -> production`~~ ❌
+  - ~~`development -> staging`~~ ❌
+  - ~~`development -> production`~~ ❌
+- Any services using the destination database will need to be stopped. You can use the `dependent_services` section of the config to list docker-compose services that should automatically be stopped and restarted.
+- Syncing will DELETE all data in the destination database and replace it with data from the source. You can make a backup before syncing using the `dump` command
 
 ### Why put this in syk?
 
@@ -37,7 +43,7 @@ Usage instructions can be viewed after installation with `sykle sync_pg_data --h
 Sync PG Data
 
 Usage:
-  syk sync_pg_data truncate --dest=<name> [--debug]
+  syk sync_pg_data recreate --dest=<name> [--debug]
   syk sync_pg_data restore --dest=<name> [--file=<name>] [--debug]
   syk sync_pg_data dump --src=<name> [--debug]
   syk sync_pg_data --src=<name> --dest=<name> [--debug]
@@ -51,14 +57,18 @@ Options:
   --file=<name>     Restore from a file
 
 Description:
-  truncate          Removes all data on db
-  restore           Restores from a file
+  recreate          Drops and then recreates a database (with data)
   dump              Dumps data to a file
+  restore           Restores from a file
 
 Example .sykle.json:
   {
      "plugins": {
         "sync_pg_data": {
+            "dependent_services": [],    // List of any services that should be
+                                         // stopped while syncing and restarted
+                                         // afterwards. (OPTIONAL)
+
             "local": {                   // Name of location
                 "env_file": ".env",      // Envfile for args to use (OPTIONAL)
                 "write": true,           // Allow writes to location
diff --git a/sykle/plugins/sync_pg_data/README.mustache b/sykle/plugins/sync_pg_data/README.mustache
@@ -2,13 +2,19 @@
 
 Wrapper around `pg_restore`, `pg_dump` and `psql` that allows you to sync data from one location to another.
 
-### Data only
-
-This command does NOT do a full database dump and restore. Dumps and restores DATA ONLY so we can sync apps that still have active connections. This means all tables in the source that you would like to link to the destination must be present.
-
-### Truncation
-
-In order to avoid foreign key constraint errors when restoring data, any existing data must be removed. This means *THE ORIGINAL DESTINATION DATA WILL BE REMOVED*.
+### Usage Instructions
+
+This command will sync BOTH data and the schema. This has a couple implications:
+
+- You will need to ensure that the place you are syncing FROM has code that is either the SAME or is OLDER than the place you are syncing to. Although you **COULD** dump data from a newer codebase into data from an older codebase, if the database schema has changed, the code and migrations will not work properly.
+  - `production -> staging` ✅
+  - `staging -> development` ✅
+  - `production -> development` ✅
+  - ~~`staging -> production`~~ ❌
+  - ~~`development -> staging`~~ ❌
+  - ~~`development -> production`~~ ❌
+- Any services using the destination database will need to be stopped. You can use the `dependent_services` section of the config to list docker-compose services that should automatically be stopped and restarted.
+- Syncing will DELETE all data in the destination database and replace it with data from the source. You can make a backup before syncing using the `dump` command
 
 ### Why put this in syk?
 
diff --git a/sykle/plugins/sync_pg_data/__init__.py b/sykle/plugins/sync_pg_data/__init__.py
@@ -1,7 +1,7 @@
 """Sync PG Data
 
 Usage:
-  syk sync_pg_data truncate --dest=<name> [--debug]
+  syk sync_pg_data recreate --dest=<name> [--debug]
   syk sync_pg_data restore --dest=<name> [--file=<name>] [--debug]
   syk sync_pg_data dump --src=<name> [--debug]
   syk sync_pg_data --src=<name> --dest=<name> [--debug]
@@ -15,14 +15,18 @@
   --file=<name>     Restore from a file
 
 Description:
-  truncate          Removes all data on db
-  restore           Restores from a file
+  recreate          Drops and then recreates a database (with data)
   dump              Dumps data to a file
+  restore           Restores from a file
 
 Example .sykle.json:
   {
      "plugins": {
         "sync_pg_data": {
+            "dependent_services": [],    // List of any services that should be
+                                         // stopped while syncing and restarted
+                                         // afterwards. (OPTIONAL)
+
             "local": {                   // Name of location
                 "env_file": ".env",      // Envfile for args to use (OPTIONAL)
                 "write": true,           // Allow writes to location
@@ -38,7 +42,7 @@
   }
 """
 
-__version__ = '0.0.3'
+__version__ = '0.1.0'
 
 from sykle.call_subprocess import call_subprocess
 from sykle.plugin_utils import IPlugin
@@ -59,7 +63,8 @@ def get_dump_file_name(self, location):
         return os.path.join(self.dump_dir, filename)
 
     def _get_location_args(self, location_name):
-        location = self.config.get(location_name)
+        locations = self.config.get("locations", {})
+        location = locations.get(location_name)
         if location is None:
             raise Exception(
                 'Unknown location "{}" (check "sync_pg_data" config)'
@@ -95,11 +100,10 @@ def dump(self, location, dump_file, debug=False):
         print('Dumping "{}" to "{}"...'.format(location, dump_file))
         call_subprocess(
             command=[
-                'pg_dump', '-C', '-h', args['HOST'],
+                'pg_dump', '-h', args['HOST'],
                 '-v', '-U', args['USER'], args['NAME'],
                 '-p', str(args.get('PORT', 5432)),
                 '-f', dump_file,
-                '--data-only',
                 '--format', 'tar'
             ],
             env={'PGPASSWORD': str(args['PASSWORD'])},
@@ -121,44 +125,45 @@ def restore(self, location, restore_file, debug=False):
                 '--username', args['USER'],
                 '--port', str(args.get('PORT', 5432)),
                 '--dbname', args['NAME'],
-                '--disable-triggers', restore_file,
-                '--data-only'
+                restore_file
             ],
             env={'PGPASSWORD': str(args['PASSWORD'])},
             debug=debug
         )
         print('Restored "{}" to "{}".'.format(restore_file, location))
 
-    def truncate(self, location, debug=False):
+    def recreate(self, location, debug=False):
         """
-        Deletes all data in the given location. (Does NOT drop tables)
+        Deletes all data in the given location.
         """
-        # Double check to ensure we aren't truncating prod
+        # Double check to ensure we aren't deleting write protected data
         self.check_write_permissions(location)
         args = self._get_location_args(location)
-        psql_command = (
-            " DO \$\$ DECLARE statements CURSOR FOR SELECT " +
-            "tablename FROM pg_tables WHERE tableowner = '{}' "
-            .format(args['NAME']) +
-            "AND schemaname = 'public'; " +
-            "BEGIN FOR stmt IN statements LOOP EXECUTE " +
-            "'TRUNCATE TABLE ' || quote_ident(stmt.tablename) || '" +
-            " CASCADE;'; END LOOP; END; \$\$ LANGUAGE plpgsql;"
-        )
 
-        print('Truncating "{}"...'.format(location))
+        print('Droping "{}"...'.format(location))
+        call_subprocess(
+            command=[
+                'dropdb', '--host', args['HOST'],
+                '--username', args['USER'],
+                '--port', str(args.get('PORT', 5432)),
+                args['NAME']
+            ],
+            env={'PGPASSWORD': str(args['PASSWORD'])},
+            debug=debug
+        )
+        print('Dropped "{}".'.format(location))
+        print('Creating "{}"...'.format(location))
         call_subprocess(
             command=[
-                'psql', '--host', args['HOST'],
+                'createdb', '--host', args['HOST'],
                 '--username', args['USER'],
                 '--port', str(args.get('PORT', 5432)),
-                '-d', args['NAME'],
-                '-c', '"{}"'.format(psql_command),
+                args['NAME']
             ],
             env={'PGPASSWORD': str(args['PASSWORD'])},
             debug=debug
         )
-        print('Truncated "{}".'.format(location))
+        print('Created "{}".'.format(location))
 
     def confirm_delete(self, location):
         return input(
@@ -182,7 +187,8 @@ def confirm_restore(self, restore_file, location):
             .format(restore_file, location)) == 'y'
 
     def check_write_permissions(self, location):
-        if not self.config.get(location).get('write'):
+        locations = self.config.get("locations", {})
+        if not locations.get(location).get('write'):
             raise Exception(
                 'Cannot delete/write data to "{}" '.format(location) +
                 '("write" is not set to true)')
@@ -197,16 +203,21 @@ def run(self):
         src = args.get('--src')
         dest = args.get('--dest')
         debug = args.get('--debug')
+        self.sykle.debug = debug
+
+        dependent_services = self.config.get("dependent_services", [])
+        for service in dependent_services:
+            self.sykle.dc(["stop", service])
 
-        if args['truncate']:
+        if args['recreate']:
             self.check_write_permissions(dest)
             if self.confirm_delete(dest):
-                self.truncate(dest, debug)
+                self.recreate(dest, debug)
         elif args['restore']:
             self.check_write_permissions(dest)
             file = args['--file'] or self.most_recent_backup()
             if self.confirm_restore(file, dest) and self.confirm_delete(dest):
-                self.truncate(dest, debug)
+                self.recreate(dest, debug)
                 self.restore(dest, file, debug)
         elif args['dump']:
             dump_file = self.get_dump_file_name(src)
@@ -217,5 +228,8 @@ def run(self):
             dump_file = self.get_dump_file_name(src)
             if self.confirm_delete(dest) and self.confirm_dump(dump_file):
                 self.dump(src, dump_file, debug)
-                self.truncate(dest, debug)
+                self.recreate(dest, debug)
                 self.restore(dest, dump_file, debug)
+
+        for service in dependent_services:
+            self.sykle.dc(["start", service])
diff --git a/test/config_test.py b/test/config_test.py
@@ -12,8 +12,8 @@ def test_docker_vars_for_depolyment(self):
                 'staging': {
                     'target': 'some-target',
                     'docker_vars': {
-                        'non_env_var': 'thing',
-                        'env_var': '$ENV_VAR'
+                        'non_env_var': u'thing',
+                        'env_var': u'$ENV_VAR'
                     }
                 }
             }
@@ -33,8 +33,8 @@ def test_for_deployment_empty(self):
 
     def test_interpolate_env_values(self):
         interpolated = Config.interpolate_env_values(
-            {'non_env_var': 'A', 'env_var': '$BUILD_NUMBER'},
-            {'BUILD_NUMBER': '1'}
+            {'non_env_var': 'A', 'env_var': u'$BUILD_NUMBER'},
+            {u'BUILD_NUMBER': u'1'}
         )
         self.assertEqual(
             interpolated,

Original file line number	Diff line number	Diff line change
`@@ -12,8 +12,8 @@ def test_docker_vars_for_depolyment(self):`
`12`	`12`	`'staging': {`
`13`	`13`	`'target': 'some-target',`
`14`	`14`	`'docker_vars': {`
`15`		`- 'non_env_var': 'thing',`
`16`		`- 'env_var': '$ENV_VAR'`
	`15`	`+ 'non_env_var': u'thing',`
	`16`	`+ 'env_var': u'$ENV_VAR'`
`17`	`17`	`}`
`18`	`18`	`}`
`19`	`19`	`}`
`@@ -33,8 +33,8 @@ def test_for_deployment_empty(self):`
`33`	`33`
`34`	`34`	`def test_interpolate_env_values(self):`
`35`	`35`	`interpolated = Config.interpolate_env_values(`
`36`		`- {'non_env_var': 'A', 'env_var': '$BUILD_NUMBER'},`
`37`		`- {'BUILD_NUMBER': '1'}`
	`36`	`+ {'non_env_var': 'A', 'env_var': u'$BUILD_NUMBER'},`
	`37`	`+ {u'BUILD_NUMBER': u'1'}`
`38`	`38`	`)`
`39`	`39`	`self.assertEqual(`
`40`	`40`	`interpolated,`