From c683483d0e6285129a1fdc65ce110034243f7e75 Mon Sep 17 00:00:00 2001
From: Mathieu Bultel <mbultel@redhat.com>
Date: Thu, 29 Sep 2016 17:25:38 +0200
Subject: [PATCH] Download templates from swift before processing with heatclient

Currently we hit an issue in update and upgrade, when heatclieant
try to process the templates and failed because there was some
missing unrendered template files, ie : post.j2.yaml

Co-Authored-By: Steven Hardy <shardy@redhat.com>
Change-Id: If95825e7df5d2c0e6cbe3575d06d57db1e8182da
Closes-Bug: #1624727
---

diff --git a/tripleoclient/tests/v1/overcloud_deploy/fakes.py b/tripleoclient/tests/v1/overcloud_deploy/fakes.py
index f5d3f89..4a256ac 100644
--- a/tripleoclient/tests/v1/overcloud_deploy/fakes.py
+++ b/tripleoclient/tests/v1/overcloud_deploy/fakes.py
@@ -107,12 +107,25 @@
 
     def __init__(self):
         self._instance = mock.Mock()
-        self.object_store = mock.Mock()
+        self.object_store = FakeObjectClient()
 
     def messaging_websocket(self, queue_name):
         return fakes.FakeWebSocket()
 
 
+class FakeObjectClient(object):
+
+    def __init__(self):
+        self._instance = mock.Mock()
+        self.put_object = mock.Mock()
+
+    def get_object(self, *args):
+        return [None, "fake"]
+
+    def get_container(self, *args):
+        return [None, [{"name": "fake"}]]
+
+
 class TestDeployOvercloud(utils.TestCommand):
 
     def setUp(self):
diff --git a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
index cb13687..0ee439b 100644
--- a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
+++ b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
@@ -15,7 +15,6 @@
 
 import fixtures
 import os
-import shutil
 import six
 import tempfile
 import yaml
@@ -54,6 +53,7 @@
 
         self.parameter_defaults_env_file = (
             tempfile.NamedTemporaryFile(mode='w', delete=False).name)
+        self.tmp_dir = self.useFixture(fixtures.TempDir())
 
     def tearDown(self):
         super(TestDeployOvercloud, self).tearDown()
@@ -79,8 +79,6 @@
     @mock.patch('tripleoclient.utils.remove_known_hosts', autospec=True)
     @mock.patch('tripleoclient.utils.wait_for_stack_ready',
                 autospec=True)
-    @mock.patch('heatclient.common.template_utils.'
-                'process_multiple_environments_and_files', autospec=True)
     @mock.patch('heatclient.common.template_utils.get_template_contents',
                 autospec=True)
     @mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@@ -94,10 +92,11 @@
     @mock.patch('tripleoclient.utils.create_keystone_credential',
                 autospec=True)
     @mock.patch('time.time', autospec=True)
-    def test_tht_scale(self, mock_time, mock_creds, mock_uuid1,
+    @mock.patch('shutil.copytree', autospec=True)
+    def test_tht_scale(self, mock_copy, mock_time, mock_creds, mock_uuid1,
                        mock_check_hypervisor_stats, mock_get_key,
                        mock_create_env, generate_certs_mock,
-                       mock_get_template_contents, mock_process_multiple_env,
+                       mock_get_template_contents,
                        wait_for_stack_ready_mock,
                        mock_remove_known_hosts, mock_keystone_initialize,
                        mock_sleep, mock_setup_endpoints,
@@ -143,8 +142,6 @@
             "id": "network id"
         }
         mock_create_env.return_value = "/fake/path"
-        mock_env = fakes.create_env()
-        mock_process_multiple_env.return_value = [{}, mock_env]
         mock_get_template_contents.return_value = [{}, "template"]
         wait_for_stack_ready_mock.return_value = True
 
@@ -221,7 +218,6 @@
             template_object=constants.OVERCLOUD_YAML_NAME)
 
         mock_create_tempest_deployer_input.assert_called_with()
-        mock_process_multiple_env.assert_called_with([])
 
     @mock.patch('tripleoclient.utils.get_overcloud_endpoint', autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
@@ -250,7 +246,7 @@
     @mock.patch('tripleoclient.utils.wait_for_stack_ready',
                 autospec=True)
     @mock.patch('heatclient.common.template_utils.'
-                'process_multiple_environments_and_files', autospec=True)
+                'process_environment_and_files', autospec=True)
     @mock.patch('heatclient.common.template_utils.get_template_contents',
                 autospec=True)
     @mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@@ -264,10 +260,13 @@
     @mock.patch('tripleoclient.utils.create_keystone_credential',
                 autospec=True)
     @mock.patch('time.time', autospec=True)
-    def test_tht_deploy(self, mock_time, mock_creds, mock_uuid1,
+    @mock.patch('shutil.copytree', autospec=True)
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    def test_tht_deploy(self, mock_tmpdir, mock_copy, mock_time, mock_creds,
+                        mock_uuid1,
                         mock_check_hypervisor_stats, mock_get_key,
                         mock_create_env, generate_certs_mock,
-                        mock_get_template_contents, mock_process_multiple_env,
+                        mock_get_template_contents, mock_process_env,
                         wait_for_stack_ready_mock,
                         mock_remove_known_hosts, mock_keystone_initialize,
                         mock_sleep, mock_setup_endpoints,
@@ -285,6 +284,7 @@
             ('ceph_storage_scale', 3)
         ]
 
+        mock_tmpdir.return_value = "/tmp/tht"
         mock_uuid1.return_value = "uuid"
         mock_creds.return_value = "key"
         mock_time.return_value = 123456789
@@ -317,7 +317,7 @@
         }
         mock_create_env.return_value = "/fake/path"
         mock_env = fakes.create_env()
-        mock_process_multiple_env.return_value = [{}, mock_env]
+        mock_process_env.return_value = [{}, mock_env]
         mock_get_template_contents.return_value = [{}, "template"]
         wait_for_stack_ready_mock.return_value = True
 
@@ -396,12 +396,12 @@
             template_object=constants.OVERCLOUD_YAML_NAME)
 
         mock_create_tempest_deployer_input.assert_called_with()
-        mock_process_multiple_env.assert_called_with(['/fake/path'])
+        mock_process_env.assert_called_with(env_path='/fake/path')
 
         mock_validate_args.assert_called_once_with(parsed_args)
 
         mock_tarball.create_tarball.assert_called_with(
-            '/usr/share/openstack-tripleo-heat-templates', mock.ANY)
+            '/tmp/tht/tripleo-heat-templates', mock.ANY)
         mock_tarball.tarball_extract_to_swift_container.assert_called_with(
             clients.tripleoclient.object_store, mock.ANY, 'overcloud')
 
@@ -423,8 +423,6 @@
     @mock.patch('tripleoclient.utils.remove_known_hosts', autospec=True)
     @mock.patch('tripleoclient.utils.wait_for_stack_ready',
                 autospec=True)
-    @mock.patch('heatclient.common.template_utils.'
-                'process_multiple_environments_and_files', autospec=True)
     @mock.patch('heatclient.common.template_utils.get_template_contents',
                 autospec=True)
     @mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@@ -434,11 +432,12 @@
     @mock.patch('tripleoclient.utils.get_config_value', autospec=True)
     @mock.patch('tripleoclient.utils.check_hypervisor_stats',
                 autospec=True)
-    def test_deploy_custom_templates(self, mock_check_hypervisor_stats,
+    @mock.patch('shutil.copytree', autospec=True)
+    def test_deploy_custom_templates(self, mock_copy,
+                                     mock_check_hypervisor_stats,
                                      mock_get_key,
                                      mock_create_env, generate_certs_mock,
                                      mock_get_template_contents,
-                                     mock_process_multiple_env,
                                      wait_for_stack_ready_mock,
                                      mock_remove_known_hosts,
                                      mock_keystone_initialize,
@@ -472,8 +471,6 @@
             "id": "network id"
         }
         mock_create_env.return_value = "/fake/path"
-        mock_env = fakes.create_env()
-        mock_process_multiple_env.return_value = [{}, mock_env]
         mock_get_template_contents.return_value = [{}, "template"]
         wait_for_stack_ready_mock.return_value = True
 
@@ -501,7 +498,6 @@
             template_object=constants.OVERCLOUD_YAML_NAME)
 
         mock_create_tempest_deployer_input.assert_called_with()
-        mock_process_multiple_env.assert_called_with([])
 
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 'set_overcloud_passwords', autospec=True)
@@ -539,7 +535,8 @@
                 '_update_parameters', autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_heat_deploy', autospec=True)
-    def test_environment_dirs(self, mock_deploy_heat,
+    @mock.patch('shutil.copytree', autospec=True)
+    def test_environment_dirs(self, mock_copy, mock_deploy_heat,
                               mock_update_parameters, mock_post_config,
                               mock_utils_check_nodes, mock_utils_endpoint,
                               mock_utils_createrc, mock_utils_tempest,
@@ -553,25 +550,29 @@
         mock_update_parameters.return_value = {}
         mock_utils_endpoint.return_value = 'foo.bar'
 
-        tmp_dir = self.useFixture(fixtures.TempDir())
-        test_env = os.path.join(tmp_dir.path, 'foo1.yaml')
+        test_env = os.path.join(self.tmp_dir.path, 'foo1.yaml')
 
         env_dirs = [os.path.join(os.environ.get('HOME', ''), '.tripleo',
-                    'environments'), tmp_dir.path]
+                    'environments'), self.tmp_dir.path]
 
         with open(test_env, 'w') as temp_file:
-            temp_file.write('#just a comment')
+            temp_file.write('resource_registry:\n  Test: OS::Heat::None')
 
-        arglist = ['--templates', '--environment-directory', tmp_dir.path]
+        arglist = ['--templates', '--environment-directory', self.tmp_dir.path]
         verifylist = [
             ('templates', '/usr/share/openstack-tripleo-heat-templates/'),
             ('environment_directories', env_dirs),
         ]
 
+        def assertEqual(*args):
+            self.assertEqual(*args)
+
         def _fake_heat_deploy(self, stack, stack_name, template_path,
                               parameters, environments, timeout, tht_root,
                               env):
-            assert test_env in environments
+            assertEqual(
+                {'parameter_defaults': {},
+                 'resource_registry': {'Test': u'OS::Heat::None'}}, env)
 
         mock_deploy_heat.side_effect = _fake_heat_deploy
 
@@ -591,7 +592,8 @@
                 '_update_parameters', autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_heat_deploy', autospec=True)
-    def test_environment_dirs_env(self, mock_deploy_heat,
+    @mock.patch('shutil.copytree', autospec=True)
+    def test_environment_dirs_env(self, mock_copy, mock_deploy_heat,
                                   mock_update_parameters, mock_post_config,
                                   mock_utils_check_nodes, mock_utils_endpoint,
                                   mock_utils_createrc, mock_utils_tempest,
@@ -605,23 +607,26 @@
         mock_update_parameters.return_value = {}
         mock_utils_endpoint.return_value = 'foo.bar'
 
-        tmp_dir = tempfile.mkdtemp()
-        test_env = os.path.join(tmp_dir, 'foo2.yaml')
-        self.addCleanup(shutil.rmtree, tmp_dir)
+        test_env = self.tmp_dir.join('foo2.yaml')
 
         with open(test_env, 'w') as temp_file:
-            temp_file.write('#just a comment')
+            temp_file.write('resource_registry:\n  Test: OS::Heat::None')
 
         arglist = ['--templates']
         verifylist = [
             ('templates', '/usr/share/openstack-tripleo-heat-templates/'),
         ]
-        os.environ['TRIPLEO_ENVIRONMENT_DIRECTORY'] = tmp_dir
+        os.environ['TRIPLEO_ENVIRONMENT_DIRECTORY'] = self.tmp_dir.path
+
+        def assertEqual(*args):
+            self.assertEqual(*args)
 
         def _fake_heat_deploy(self, stack, stack_name, template_path,
                               parameters, environments, timeout, tht_root,
                               env):
-            assert test_env in environments
+            assertEqual(
+                {'parameter_defaults': {},
+                 'resource_registry': {'Test': u'OS::Heat::None'}}, env)
 
         mock_deploy_heat.side_effect = _fake_heat_deploy
 
@@ -636,7 +641,8 @@
     @mock.patch('tripleoclient.utils.get_overcloud_endpoint', autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_deploy_tripleo_heat_templates', autospec=True)
-    def test_rhel_reg_params_provided(self, mock_deploy_tht,
+    @mock.patch('shutil.copytree', autospec=True)
+    def test_rhel_reg_params_provided(self, mock_copytree, mock_deploy_tht,
                                       mock_oc_endpoint,
                                       mock_create_ocrc,
                                       mock_set_oc_passwords,
@@ -682,7 +688,7 @@
     @mock.patch('tripleoclient.utils.wait_for_stack_ready',
                 autospec=True)
     @mock.patch('heatclient.common.template_utils.'
-                'process_multiple_environments_and_files', autospec=True)
+                'process_environment_and_files', autospec=True)
     @mock.patch('heatclient.common.template_utils.get_template_contents',
                 autospec=True)
     @mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@@ -692,11 +698,15 @@
     @mock.patch('tripleoclient.utils.get_config_value', autospec=True)
     @mock.patch('tripleoclient.utils.check_hypervisor_stats',
                 autospec=True)
-    def test_deploy_rhel_reg(self, mock_check_hypervisor_stats,
+    @mock.patch('shutil.copytree', autospec=True)
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    @mock.patch('shutil.rmtree', autospec=True)
+    def test_deploy_rhel_reg(self, mock_rmtree, mock_tmpdir, mock_copy,
+                             mock_check_hypervisor_stats,
                              mock_get_key,
                              mock_create_env, generate_certs_mock,
                              mock_get_template_contents,
-                             mock_process_multiple_env,
+                             mock_process_env,
                              wait_for_stack_ready_mock,
                              mock_remove_known_hosts,
                              mock_keystone_initialize,
@@ -721,8 +731,10 @@
             ('reg_activation_key', 'super-awesome-key')
         ]
 
+        mock_tmpdir.return_value = None
+        mock_tmpdir.return_value = '/tmp/tht'
         mock_generate_overcloud_passwords.return_value = self._get_passwords()
-        mock_process_multiple_env.return_value = [{}, fakes.create_env()]
+        mock_process_env.return_value = [{}, fakes.create_env()]
         mock_get_template_contents.return_value = [{}, "template"]
         wait_for_stack_ready_mock.return_value = True
 
@@ -753,15 +765,14 @@
 
         self.cmd.take_action(parsed_args)
 
-        args, kwargs = mock_process_multiple_env.call_args
-        self.assertIn(
-            '/usr/share/openstack-tripleo-heat-templates/extraconfig/pre_dep'
-            'loy/rhel-registration/rhel-registration-resource-registry.yaml',
-            args[0])
-        self.assertIn(
-            '/usr/share/openstack-tripleo-heat-templates/extraconfig/pre_dep'
-            'loy/rhel-registration/environment-rhel-registration.yaml',
-            args[0])
+        tht_prefix = ('/tmp/tht/tripleo-heat-templates/extraconfig/'
+                      'pre_deploy/rhel-registration/')
+        calls = [
+            mock.call(env_path=tht_prefix +
+                      'rhel-registration-resource-registry.yaml'),
+            mock.call(env_path=tht_prefix +
+                      'environment-rhel-registration.yaml')]
+        mock_process_env.assert_has_calls(calls)
 
     def test_validate_args_correct(self):
         arglist = ['--templates',
@@ -792,7 +803,9 @@
                           self.cmd._validate_args,
                           parsed_args)
 
-    def test_validate_args_missing_environment_files(self):
+    @mock.patch('tripleoclient.tests.v1.overcloud_deploy.fakes.'
+                'FakeObjectClient.get_object', autospec=True)
+    def test_validate_args_missing_environment_files(self, mock_obj):
         arglist = ['--templates',
                    '-e', 'nonexistent.yaml']
         verifylist = [
@@ -800,12 +813,33 @@
             ('environment_files', ['nonexistent.yaml']),
         ]
 
+        mock_obj.side_effect = ObjectClientException(mock.Mock(
+                                                     '/fake/path not found'))
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
 
         self.assertRaises(oscexc.CommandError,
                           self.cmd._validate_args,
                           parsed_args)
 
+    @mock.patch('os.path.isfile', autospec=True)
+    def test_validate_args_missing_rendered_files(self, mock_isfile):
+        tht_path = '/usr/share/openstack-tripleo-heat-templates/'
+        env_path = os.path.join(tht_path, 'noexist.yaml')
+        arglist = ['--templates',
+                   '-e', env_path]
+        verifylist = [
+            ('templates', '/usr/share/openstack-tripleo-heat-templates/'),
+            ('environment_files', [env_path]),
+        ]
+
+        mock_isfile.side_effect = [False, True]
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        self.cmd._validate_args(parsed_args)
+        calls = [mock.call(env_path),
+                 mock.call(env_path.replace(".yaml", ".j2.yaml"))]
+        mock_isfile.assert_has_calls(calls)
+
     def test_validate_args_no_tunnel_type(self):
         arglist = ['--templates',
                    '--neutron-network-type', 'nettype']
@@ -953,7 +987,10 @@
                 '_heat_deploy', autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 'set_overcloud_passwords', autospec=True)
-    def test_answers_file(self,
+    @mock.patch('shutil.copytree', autospec=True)
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    @mock.patch('shutil.rmtree', autospec=True)
+    def test_answers_file(self, mock_rmtree, mock_tmpdir, mock_copy,
                           mock_set_overcloud_passwords,
                           mock_heat_deploy,
                           mock_oc_endpoint,
@@ -966,32 +1003,41 @@
         workflow_client.action_executions.create.return_value = mock.MagicMock(
             output='{"result":[]}')
 
+        mock_tmpdir.return_value = self.tmp_dir.path
+        mock_rmtree.return_value = None
         network_client = clients.network
         network_client.stacks.get.return_value = None
         net = network_client.api.find_attr('networks', 'ctlplane')
         net.configure_mock(__getitem__=lambda x, y: 'testnet')
 
-        with tempfile.NamedTemporaryFile(mode="w+t") as answerfile:
-            with open('/tmp/environment.yaml', "w+t") as environmentfile:
+        test_env = self.tmp_dir.join('foo1.yaml')
+        with open(test_env, 'w') as temp_file:
+            temp_file.write('resource_registry:\n  Test: OS::Heat::None')
+
+        test_env2 = self.tmp_dir.join('foo2.yaml')
+        with open(test_env2, 'w') as temp_file:
+            temp_file.write('resource_registry:\n  Test2: OS::Heat::None')
+
+        test_answerfile = self.tmp_dir.join('answerfile')
+        with open(test_answerfile, 'w') as answerfile:
                 yaml.dump(
-                    {'templates': '/dev/null',
-                     'environments': ['/tmp/foo3.yaml']
+                    {'templates':
+                     '/usr/share/openstack-tripleo-heat-templates/',
+                     'environments': [test_env]
                      },
                     answerfile
                 )
-                answerfile.flush()
 
-                arglist = ['--answers-file', answerfile.name,
-                           '--environment-file', environmentfile.name,
-                           '--block-storage-scale', '3']
-                verifylist = [
-                    ('answers_file', answerfile.name),
-                    ('environment_files', [environmentfile.name]),
-                    ('block_storage_scale', 3)
-                ]
-                parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+        arglist = ['--answers-file', test_answerfile,
+                   '--environment-file', test_env2,
+                   '--block-storage-scale', '3']
+        verifylist = [
+            ('answers_file', test_answerfile),
+            ('environment_files', [test_env2]),
+            ('block_storage_scale', 3)]
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
 
-                self.cmd.take_action(parsed_args)
+        self.cmd.take_action(parsed_args)
 
         self.assertTrue(mock_heat_deploy.called)
         self.assertTrue(mock_oc_endpoint.called)
@@ -1000,12 +1046,15 @@
 
         # Check that Heat was called with correct parameters:
         call_args = mock_heat_deploy.call_args[0]
-        self.assertEqual(call_args[3], '/dev/null/overcloud.yaml')
-        self.assertIn('/tmp/foo3.yaml', call_args[5])
-        self.assertIn('/tmp/environment.yaml', call_args[5])
-        foo_index = call_args[5].index('/tmp/foo3.yaml')
-        env_index = call_args[5].index('/tmp/environment.yaml')
-        self.assertGreater(env_index, foo_index)
+        self.assertEqual(call_args[3],
+                         self.tmp_dir.join(
+                             'tripleo-heat-templates/overcloud.yaml'))
+        self.assertEqual(call_args[7],
+                         self.tmp_dir.join('tripleo-heat-templates'))
+        self.assertIn('Test', call_args[8]['resource_registry'])
+        self.assertIn('Test2', call_args[8]['resource_registry'])
+        self.assertEqual(
+            3, call_args[8]['parameter_defaults']['BlockStorageCount'])
 
         mock_create_tempest_deployer_input.assert_called_with()
 
@@ -1101,16 +1150,18 @@
     @mock.patch('tripleoclient.utils.generate_overcloud_passwords')
     @mock.patch('tripleoclient.utils.create_overcloudrc')
     @mock.patch('heatclient.common.template_utils.'
-                'process_multiple_environments_and_files', autospec=True)
+                'process_environment_and_files', autospec=True)
     @mock.patch('heatclient.common.template_utils.get_template_contents',
                 autospec=True)
     @mock.patch('tripleoclient.utils.create_environment_file',
                 autospec=True)
     @mock.patch('tripleoclient.utils.get_config_value', autospec=True)
-    def test_ntp_server_mandatory(self, mock_get_key,
+    @mock.patch('shutil.copytree', autospec=True)
+    def test_ntp_server_mandatory(self, mock_copy,
+                                  mock_get_key,
                                   mock_create_env,
                                   mock_get_template_contents,
-                                  mock_process_multiple_env,
+                                  mock_process_env,
                                   mock_create_overcloudrc,
                                   mock_generate_overcloud_passwords,
                                   mock_create_parameters_env,
@@ -1141,7 +1192,7 @@
 
         mock_create_env.return_value = "/fake/path"
         mock_env = fakes.create_env()
-        mock_process_multiple_env.return_value = [{}, mock_env]
+        mock_process_env.return_value = [{}, mock_env]
         mock_get_template_contents.return_value = [{}, "template"]
 
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
@@ -1174,7 +1225,7 @@
     @mock.patch('tripleoclient.utils.wait_for_stack_ready',
                 autospec=True)
     @mock.patch('heatclient.common.template_utils.'
-                'process_multiple_environments_and_files', autospec=True)
+                'process_environment_and_files', autospec=True)
     @mock.patch('heatclient.common.template_utils.get_template_contents',
                 autospec=True)
     @mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@@ -1188,12 +1239,14 @@
     @mock.patch('tripleoclient.utils.create_keystone_credential',
                 autospec=True)
     @mock.patch('time.time', autospec=True)
-    def test_tht_deploy_with_ntp(self, mock_time, mock_creds, mock_uuid1,
+    @mock.patch('shutil.copytree', autospec=True)
+    def test_tht_deploy_with_ntp(self, mock_copy, mock_time, mock_creds,
+                                 mock_uuid1,
                                  mock_check_hypervisor_stats,
                                  mock_get_key, mock_create_env,
                                  generate_certs_mock,
                                  mock_get_template_contents,
-                                 mock_process_multiple_env,
+                                 mock_process_env,
                                  wait_for_stack_ready_mock,
                                  mock_remove_known_hosts,
                                  mock_keystone_initialize,
@@ -1253,7 +1306,7 @@
         }
         mock_create_env.return_value = "/fake/path"
         mock_env = fakes.create_env_with_ntp()
-        mock_process_multiple_env.return_value = [{}, mock_env]
+        mock_process_env.return_value = [{}, mock_env]
         mock_get_template_contents.return_value = [{}, "template"]
         wait_for_stack_ready_mock.return_value = True
 
@@ -1328,6 +1381,6 @@
             template_object=constants.OVERCLOUD_YAML_NAME)
 
         mock_create_tempest_deployer_input.assert_called_with()
-        mock_process_multiple_env.assert_called_with(['/fake/path'])
+        mock_process_env.assert_called_with(env_path='/fake/path')
 
         mock_validate_args.assert_called_once_with(parsed_args)
diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py
index bbbcda7..cb98a2a 100644
--- a/tripleoclient/v1/overcloud_deploy.py
+++ b/tripleoclient/v1/overcloud_deploy.py
@@ -21,12 +21,15 @@
 import os
 import os.path
 import re
+import shutil
 import six
+import tempfile
 import time
 import uuid
 import yaml
 
 from heatclient.common import template_utils
+from heatclient import exc as hc_exc
 from keystoneclient import exceptions as kscexc
 from os_cloud_config import keystone
 from os_cloud_config import keystone_pki
@@ -219,22 +222,46 @@
         parameter_defaults = {"parameter_defaults": parameters}
         return parameter_defaults
 
+    def _process_multiple_environments(self, created_env_files, added_files,
+                                       tht_root, user_tht_root):
+        env_files = {}
+        localenv = {}
+        for env_path in created_env_files:
+            self.log.debug("Processing environment files %s" % env_path)
+            abs_env_path = os.path.abspath(env_path)
+            if abs_env_path.startswith(user_tht_root):
+                new_env_path = abs_env_path.replace(user_tht_root, tht_root)
+                self.log.debug("Redirecting env file %s to %s"
+                               % (abs_env_path, new_env_path))
+                env_path = new_env_path
+            try:
+                files, env = template_utils.process_environment_and_files(
+                    env_path=env_path)
+            except hc_exc.CommandError as ex:
+                self.log.debug("Error %s processing environment file %s"
+                               % (six.text_type(ex), env_path))
+                # FIXME(shardy) We need logic here to handle the case described
+                # in https://bugs.launchpad.net/tripleo/+bug/1625783 so that
+                # resource_registry contents are redirected to tmpdir tht_root
+                raise
+            if files:
+                self.log.debug("Adding files %s for %s" % (files, env_path))
+                env_files.update(files)
+
+            # 'env' can be a deeply nested dictionary, so a simple update is
+            # not enough
+            localenv = template_utils.deep_update(localenv, env)
+        return env_files, localenv
+
     def _heat_deploy(self, stack, stack_name, template_path, parameters,
-                     created_env_files, timeout, tht_root, env):
+                     env_files, timeout, tht_root, env):
         """Verify the Baremetal nodes are available and do a stack update"""
 
         clients = self.app.client_manager
         workflow_client = clients.workflow_engine
 
-        self.log.debug("Processing environment files %s" % created_env_files)
-        env_files, localenv = (
-            template_utils.process_multiple_environments_and_files(
-                created_env_files))
-        # Command line has more precedence than env files
-        template_utils.deep_update(localenv, env)
-
         if stack:
-            update.add_breakpoints_cleanup_into_env(localenv)
+            update.add_breakpoints_cleanup_into_env(env)
 
         self.log.debug("Getting template contents from plan %s" % stack_name)
         # We need to reference the plan here, not the local
@@ -259,7 +286,7 @@
 
         number_controllers = int(parameters.get('ControllerCount', 0))
         if number_controllers > 1:
-            if not localenv.get('parameter_defaults').get('NtpServer'):
+            if not env.get('parameter_defaults').get('NtpServer'):
                 raise exceptions.InvalidConfiguration(
                     'Specify --ntp-server as parameter or NtpServer in '
                     'environments when using multiple controllers '
@@ -270,7 +297,7 @@
         moved_files = self._upload_missing_files(
             stack_name, objectclient, files, tht_root)
         self._process_and_upload_environment(
-            stack_name, objectclient, localenv, moved_files, tht_root,
+            stack_name, objectclient, env, moved_files, tht_root,
             workflow_client)
 
         deployment.deploy_and_wait(self.log, clients, stack, stack_name,
@@ -381,7 +408,47 @@
 
         return file_relocation
 
-    def _deploy_tripleo_heat_templates(self, stack, parsed_args):
+    def _download_missing_files_from_plan(self, tht_dir, plan_name):
+        # get and download missing files into tmp directory
+        clients = self.app.client_manager
+        objectclient = clients.tripleoclient.object_store
+        plan_list = objectclient.get_container(plan_name)
+        plan_filenames = [f['name'] for f in plan_list[1]]
+        added_files = {}
+        for pf in plan_filenames:
+            file_path = os.path.join(tht_dir, pf)
+            if not os.path.isfile(file_path):
+                self.log.debug("Missing in templates directory, downloading \
+                               %s from swift into %s" % (pf, file_path))
+                if not os.path.exists(os.path.dirname(file_path)):
+                    os.makedirs(os.path.dirname(file_path))
+                with open(file_path, 'w') as f:
+                    f.write(objectclient.get_object(plan_name, pf)[1])
+                added_files[pf] = file_path
+        self.log.debug("added_files = %s" % added_files)
+        return added_files
+
+    def _deploy_tripleo_heat_templates_tmpdir(self, stack, parsed_args):
+        # copy tht_root to temporary directory because we need to
+        # download any missing (e.g j2 rendered) files from the plan
+        tht_root = os.path.abspath(parsed_args.templates)
+        tht_tmp = tempfile.mkdtemp(prefix='tripleoclient-')
+        new_tht_root = "%s/tripleo-heat-templates" % tht_tmp
+        self.log.debug("Creating temporary templates tree in %s"
+                       % new_tht_root)
+        try:
+            shutil.copytree(tht_root, new_tht_root, symlinks=True)
+            self._deploy_tripleo_heat_templates(stack, parsed_args,
+                                                new_tht_root, tht_root)
+        finally:
+            if parsed_args.no_cleanup:
+                self.log.warning("Not cleaning temporary directory %s"
+                                 % tht_tmp)
+            else:
+                shutil.rmtree(tht_tmp)
+
+    def _deploy_tripleo_heat_templates(self, stack, parsed_args,
+                                       tht_root, user_tht_root):
         """Deploy the fixed templates in TripleO Heat Templates"""
         clients = self.app.client_manager
         network_client = clients.network
@@ -390,8 +457,6 @@
         parameters = self._update_parameters(
             parsed_args, network_client, stack)
 
-        tht_root = os.path.abspath(parsed_args.templates)
-
         plans = plan_management.list_deployment_plans(workflow_client)
 
         # TODO(d0ugal): We need to put a more robust strategy in place here to
@@ -405,6 +470,10 @@
             plan_management.create_plan_from_templates(
                 clients, parsed_args.stack, tht_root, parsed_args.roles_file)
 
+        # Get any missing (e.g j2 rendered) files from the plan to tht_root
+        added_files = self._download_missing_files_from_plan(
+            tht_root, parsed_args.stack)
+
         print("Deploying templates in the directory {0}".format(
             os.path.abspath(tht_root)))
 
@@ -433,18 +502,25 @@
         if parsed_args.environment_files:
             created_env_files.extend(parsed_args.environment_files)
 
+        self.log.debug("Processing environment files %s" % created_env_files)
+        env_files, localenv = self._process_multiple_environments(
+            created_env_files, added_files, tht_root, user_tht_root)
+
+        # Command line has more precedence than env files
+        template_utils.deep_update(localenv, env)
+
         self._try_overcloud_deploy_with_compat_yaml(
-            tht_root, stack, parsed_args.stack, parameters, created_env_files,
-            parsed_args.timeout, env)
+            tht_root, stack, parsed_args.stack, parameters, env_files,
+            parsed_args.timeout, localenv)
 
     def _try_overcloud_deploy_with_compat_yaml(self, tht_root, stack,
                                                stack_name, parameters,
-                                               created_env_files, timeout,
+                                               env_files, timeout,
                                                env):
         overcloud_yaml = os.path.join(tht_root, constants.OVERCLOUD_YAML_NAME)
         try:
             self._heat_deploy(stack, stack_name, overcloud_yaml,
-                              parameters, created_env_files, timeout,
+                              parameters, env_files, timeout,
                               tht_root, env)
         except ClientException as e:
             messages = 'Failed to deploy: %s' % str(e)
@@ -631,7 +707,11 @@
             nonexisting_envs = []
             for env_file in parsed_args.environment_files:
                 if not os.path.isfile(env_file):
-                    nonexisting_envs.append(env_file)
+                    # Tolerate missing file if there's a j2.yaml file that will
+                    # be rendered in the plan but not available locally (yet)
+                    if not os.path.isfile(env_file.replace(".yaml",
+                                                           ".j2.yaml")):
+                        nonexisting_envs.append(env_file)
             if nonexisting_envs:
                 raise oscexc.CommandError(
                     "Error: The following files were not found: {0}".format(
@@ -948,6 +1028,10 @@
             help=_('Roles file, overrides the default %s in the --templates '
                    'directory') % constants.OVERCLOUD_ROLES_FILE
         )
+        parser.add_argument(
+            '--no-cleanup', action='store_true',
+            help=_('Don\'t cleanup temporary files, just log their location')
+        )
         # TODO(bnemec): In Ocata or later, remove this group and just leave
         # --validation-errors-nonfatal
         error_group = parser.add_mutually_exclusive_group()
@@ -1101,7 +1185,7 @@
             print("Validation Finished")
             return
 
-        self._deploy_tripleo_heat_templates(stack, parsed_args)
+        self._deploy_tripleo_heat_templates_tmpdir(stack, parsed_args)
 
         # Get a new copy of the stack after stack update/create. If it was
         # a create then the previous stack object would be None.