Re: [PATCH 4/5] tree-inline: implement SIMT privatization, part 3

2017-03-23 Thread Jakub Jelinek 
On Thu, Mar 23, 2017 at 08:00:11PM +0300, Alexander Monakov wrote:
> On Thu, 23 Mar 2017, Jakub Jelinek wrote:
> > And then clear it.  That doesn't look like the right thing.
> > 
> > So either you need some bool variable whether you've actually allocated
> > the vector in the current expand_call_inline and use that instead of
> > if (id->dst_simt_vars), or maybe you should clear id->dst_simt_vars
> > otherwise and save/restore it around unconditionally.
> 
> Yes, thanks for catching this.  I went for the latter approach in the 
> following
> patch.

Ok for trunk, thanks.

For the nvptx bits, I think you need to ask Bernd to review it.

Jakub

Re: [PATCH 4/5] tree-inline: implement SIMT privatization, part 3

2017-03-23 Thread Alexander Monakov 
On Thu, 23 Mar 2017, Jakub Jelinek wrote:
> And then clear it.  That doesn't look like the right thing.
> 
> So either you need some bool variable whether you've actually allocated
> the vector in the current expand_call_inline and use that instead of
> if (id->dst_simt_vars), or maybe you should clear id->dst_simt_vars
> otherwise and save/restore it around unconditionally.

Yes, thanks for catching this.  I went for the latter approach in the following
patch.

---
 gcc/tree-inline.c | 61 ---
 gcc/tree-inline.h |  4 
 2 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index 6b6d489..b3bb3d6 100644
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -4385,6 +4385,11 @@ expand_call_inline (basic_block bb, gimple *stmt, 
copy_body_data *id)
   gcall *call_stmt;
   unsigned int i;
   unsigned int prop_mask, src_properties;
+  struct function *dst_cfun;
+  tree simduid;
+  use_operand_p use;
+  gimple *simtenter_stmt = NULL;
+  vec *simtvars_save;
 
   /* The gimplifier uses input_location in too many places, such as
  internal_get_tmp_var ().  */
@@ -4588,15 +4593,26 @@ expand_call_inline (basic_block bb, gimple *stmt, 
copy_body_data *id)
   id->src_cfun = DECL_STRUCT_FUNCTION (fn);
   id->call_stmt = call_stmt;
 
+  /* When inlining into an OpenMP SIMD-on-SIMT loop, arrange for new automatic
+ variables to be added to IFN_GOMP_SIMT_ENTER argument list.  */
+  dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
+  simtvars_save = id->dst_simt_vars;
+  if (!(dst_cfun->curr_properties & PROP_gimple_lomp_dev)
+  && (simduid = bb->loop_father->simduid) != NULL_TREE
+  && (simduid = ssa_default_def (dst_cfun, simduid)) != NULL_TREE
+  && single_imm_use (simduid, , _stmt)
+  && is_gimple_call (simtenter_stmt)
+  && gimple_call_internal_p (simtenter_stmt, IFN_GOMP_SIMT_ENTER))
+vec_alloc (id->dst_simt_vars, 0);
+  else
+id->dst_simt_vars = NULL;
+
   /* If the src function contains an IFN_VA_ARG, then so will the dst
  function after inlining.  Likewise for IFN_GOMP_USE_SIMT.  */
   prop_mask = PROP_gimple_lva | PROP_gimple_lomp_dev;
   src_properties = id->src_cfun->curr_properties & prop_mask;
   if (src_properties != prop_mask)
-{
-  struct function *dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
-  dst_cfun->curr_properties &= src_properties | ~prop_mask;
-}
+dst_cfun->curr_properties &= src_properties | ~prop_mask;
 
   gcc_assert (!id->src_cfun->after_inlining);
 
@@ -4730,6 +4746,27 @@ expand_call_inline (basic_block bb, gimple *stmt, 
copy_body_data *id)
   if (cfun->gimple_df)
 pt_solution_reset (>gimple_df->escaped);
 
+  /* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments.  */
+  if (id->dst_simt_vars && id->dst_simt_vars->length () > 0)
+{
+  size_t nargs = gimple_call_num_args (simtenter_stmt);
+  vec *vars = id->dst_simt_vars;
+  auto_vec newargs (nargs + vars->length ());
+  for (size_t i = 0; i < nargs; i++)
+   newargs.quick_push (gimple_call_arg (simtenter_stmt, i));
+  for (tree *pvar = vars->begin (); pvar != vars->end (); pvar++)
+   {
+ tree ptrtype = build_pointer_type (TREE_TYPE (*pvar));
+ newargs.quick_push (build1 (ADDR_EXPR, ptrtype, *pvar));
+   }
+  gcall *g = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, newargs);
+  gimple_call_set_lhs (g, gimple_call_lhs (simtenter_stmt));
+  gimple_stmt_iterator gsi = gsi_for_stmt (simtenter_stmt);
+  gsi_replace (, g, false);
+}
+  vec_free (id->dst_simt_vars);
+  id->dst_simt_vars = simtvars_save;
+
   /* Clean up.  */
   if (id->debug_map)
 {
@@ -5453,9 +5490,19 @@ copy_decl_for_dup_finish (copy_body_data *id, tree decl, 
tree copy)
function.  */
 ;
   else
-/* Ordinary automatic local variables are now in the scope of the
-   new function.  */
-DECL_CONTEXT (copy) = id->dst_fn;
+{
+  /* Ordinary automatic local variables are now in the scope of the
+new function.  */
+  DECL_CONTEXT (copy) = id->dst_fn;
+  if (VAR_P (copy) && id->dst_simt_vars && !is_gimple_reg (copy))
+   {
+ if (!lookup_attribute ("omp simt private", DECL_ATTRIBUTES (copy)))
+   DECL_ATTRIBUTES (copy)
+ = tree_cons (get_identifier ("omp simt private"), NULL,
+  DECL_ATTRIBUTES (copy));
+ id->dst_simt_vars->safe_push (copy);
+   }
+}
 
   return copy;
 }
diff --git a/gcc/tree-inline.h b/gcc/tree-inline.h
index 88b3286..ffb8333 100644
--- a/gcc/tree-inline.h
+++ b/gcc/tree-inline.h
@@ -145,6 +145,10 @@ struct copy_body_data
  equivalents in the function into which it is being inlined.  */
   hash_map *dependence_map;
 
+  /* A list of addressable local variables remapped into the caller
+ when inlining a call within an OpenMP SIMD-on-SIMT loop.  */
+  vec

Re: [PATCH 4/5] tree-inline: implement SIMT privatization, part 3

2017-03-23 Thread Jakub Jelinek 
On Thu, Mar 23, 2017 at 07:15:52PM +0300, Alexander Monakov wrote:
>   * tree-inline.h (struct copy_body_data): New field dst_simt_vars.
> * tree-inline.c (expand_call_inline): Handle SIMT privatization.
> (copy_decl_for_dup_finish): Ditto.
> ---
>  gcc/tree-inline.c | 65 
> +--
>  gcc/tree-inline.h |  4 
>  2 files changed, 62 insertions(+), 7 deletions(-)
> 
> @@ -4588,15 +4593,26 @@ expand_call_inline (basic_block bb, gimple *stmt, 
> copy_body_data *id)
>id->src_cfun = DECL_STRUCT_FUNCTION (fn);
>id->call_stmt = call_stmt;
>  
> +  /* When inlining into an OpenMP SIMD-on-SIMT loop, arrange for new 
> automatic
> + variables to be added to IFN_GOMP_SIMT_ENTER argument list.  */
> +  dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
> +  if (!(dst_cfun->curr_properties & PROP_gimple_lomp_dev)
> +  && (simduid = bb->loop_father->simduid) != NULL_TREE
> +  && (simduid = ssa_default_def (dst_cfun, simduid)) != NULL_TREE
> +  && single_imm_use (simduid, , _stmt)
> +  && is_gimple_call (simtenter_stmt)
> +  && gimple_call_internal_p (simtenter_stmt, IFN_GOMP_SIMT_ENTER))
> +{
> +  simtvars_st = id->dst_simt_vars;
> +  vec_alloc (id->dst_simt_vars, 0);
> +}

One more thing.  If the above if condition is false, you keep
id->dst_simt_vars what it was (which means simtvars_st is NULL).
If it was non-NULL already, then:

> @@ -4730,6 +4746,31 @@ expand_call_inline (basic_block bb, gimple *stmt, 
> copy_body_data *id)
>if (cfun->gimple_df)
>  pt_solution_reset (>gimple_df->escaped);
>  
> +  /* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments.  */
> +  if (id->dst_simt_vars)
> +{

This will be true.

> +  if (id->dst_simt_vars->length () > 0)
> + {
> +   size_t nargs = gimple_call_num_args (simtenter_stmt);
> +   vec *vars = id->dst_simt_vars;
> +   auto_vec newargs (nargs + vars->length ());
> +   for (size_t i = 0; i < nargs; i++)
> + newargs.quick_push (gimple_call_arg (simtenter_stmt, i));
> +   for (tree *pvar = vars->begin (); pvar != vars->end (); pvar++)
> + {
> +   tree ptrtype = build_pointer_type (TREE_TYPE (*pvar));
> +   newargs.quick_push (build1 (ADDR_EXPR, ptrtype, *pvar));
> + }
> +   gcall *g
> + = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, newargs);
> +   gimple_call_set_lhs (g, gimple_call_lhs (simtenter_stmt));
> +   gimple_stmt_iterator gsi = gsi_for_stmt (simtenter_stmt);
> +   gsi_replace (, g, false);
> + }

And you handle dst_simt_vars from some other invocation.

> +  vec_free (id->dst_simt_vars);
> +  id->dst_simt_vars = simtvars_st;

And then clear it.  That doesn't look like the right thing.

So either you need some bool variable whether you've actually allocated
the vector in the current expand_call_inline and use that instead of
if (id->dst_simt_vars), or maybe you should clear id->dst_simt_vars
otherwise and save/restore it around unconditionally.

Jakub

Re: [PATCH 4/5] tree-inline: implement SIMT privatization, part 3

2017-03-23 Thread Alexander Monakov 
On Thu, 23 Mar 2017, Jakub Jelinek wrote:
> > Sorry for missing the IR stability issue.  This code relies on dst_simt_vars
> > being a set and thus having no duplicate entries (so the implicit lookup 
> > when
> > adding an element is needed).
> > 
> > However, I think I was overly cautious: looking again, I think we can't 
> > enter
> > copy_decl_for_dup_finish twice with the same 'copy' VAR_DECL.  Changing it 
> > to a
> > vec should be fine then?
> 
> Yeah, callers of copy_decl* should look the orig var in the decl map first,
> plus if you look at all copy_decl_for_dup_finish callers, all of them first
> create a new tree (usually copy_node) and then pass it as copy to
> copy_decl_for_dup_finish, so you'll never get the same copy in there.
> 
> So, please change it into a vector.

Thanks — here's the updated patch.  I've also noticed there's no need to rebuild
the existing SIMT_ENTER statement if we didn't add any new privatized variables.

* tree-inline.h (struct copy_body_data): New field dst_simt_vars.
* tree-inline.c (expand_call_inline): Handle SIMT privatization.
(copy_decl_for_dup_finish): Ditto.
---
 gcc/tree-inline.c | 65 +--
 gcc/tree-inline.h |  4 
 2 files changed, 62 insertions(+), 7 deletions(-)

diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index 6b6d489..a84e569 100644
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -4385,6 +4385,11 @@ expand_call_inline (basic_block bb, gimple *stmt, 
copy_body_data *id)
   gcall *call_stmt;
   unsigned int i;
   unsigned int prop_mask, src_properties;
+  struct function *dst_cfun;
+  tree simduid;
+  use_operand_p use;
+  gimple *simtenter_stmt = NULL;
+  vec *simtvars_st = NULL;
 
   /* The gimplifier uses input_location in too many places, such as
  internal_get_tmp_var ().  */
@@ -4588,15 +4593,26 @@ expand_call_inline (basic_block bb, gimple *stmt, 
copy_body_data *id)
   id->src_cfun = DECL_STRUCT_FUNCTION (fn);
   id->call_stmt = call_stmt;
 
+  /* When inlining into an OpenMP SIMD-on-SIMT loop, arrange for new automatic
+ variables to be added to IFN_GOMP_SIMT_ENTER argument list.  */
+  dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
+  if (!(dst_cfun->curr_properties & PROP_gimple_lomp_dev)
+  && (simduid = bb->loop_father->simduid) != NULL_TREE
+  && (simduid = ssa_default_def (dst_cfun, simduid)) != NULL_TREE
+  && single_imm_use (simduid, , _stmt)
+  && is_gimple_call (simtenter_stmt)
+  && gimple_call_internal_p (simtenter_stmt, IFN_GOMP_SIMT_ENTER))
+{
+  simtvars_st = id->dst_simt_vars;
+  vec_alloc (id->dst_simt_vars, 0);
+}
+
   /* If the src function contains an IFN_VA_ARG, then so will the dst
  function after inlining.  Likewise for IFN_GOMP_USE_SIMT.  */
   prop_mask = PROP_gimple_lva | PROP_gimple_lomp_dev;
   src_properties = id->src_cfun->curr_properties & prop_mask;
   if (src_properties != prop_mask)
-{
-  struct function *dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
-  dst_cfun->curr_properties &= src_properties | ~prop_mask;
-}
+dst_cfun->curr_properties &= src_properties | ~prop_mask;
 
   gcc_assert (!id->src_cfun->after_inlining);
 
@@ -4730,6 +4746,31 @@ expand_call_inline (basic_block bb, gimple *stmt, 
copy_body_data *id)
   if (cfun->gimple_df)
 pt_solution_reset (>gimple_df->escaped);
 
+  /* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments.  */
+  if (id->dst_simt_vars)
+{
+  if (id->dst_simt_vars->length () > 0)
+   {
+ size_t nargs = gimple_call_num_args (simtenter_stmt);
+ vec *vars = id->dst_simt_vars;
+ auto_vec newargs (nargs + vars->length ());
+ for (size_t i = 0; i < nargs; i++)
+   newargs.quick_push (gimple_call_arg (simtenter_stmt, i));
+ for (tree *pvar = vars->begin (); pvar != vars->end (); pvar++)
+   {
+ tree ptrtype = build_pointer_type (TREE_TYPE (*pvar));
+ newargs.quick_push (build1 (ADDR_EXPR, ptrtype, *pvar));
+   }
+ gcall *g
+   = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, newargs);
+ gimple_call_set_lhs (g, gimple_call_lhs (simtenter_stmt));
+ gimple_stmt_iterator gsi = gsi_for_stmt (simtenter_stmt);
+ gsi_replace (, g, false);
+   }
+  vec_free (id->dst_simt_vars);
+  id->dst_simt_vars = simtvars_st;
+}
+
   /* Clean up.  */
   if (id->debug_map)
 {
@@ -5453,9 +5494,19 @@ copy_decl_for_dup_finish (copy_body_data *id, tree decl, 
tree copy)
function.  */
 ;
   else
-/* Ordinary automatic local variables are now in the scope of the
-   new function.  */
-DECL_CONTEXT (copy) = id->dst_fn;
+{
+  /* Ordinary automatic local variables are now in the scope of the
+new function.  */
+  DECL_CONTEXT (copy) = id->dst_fn;
+  if (VAR_P (copy) && id->dst_simt_vars && !is_gimple_reg (copy))
+   {
+ if

Re: [PATCH 4/5] tree-inline: implement SIMT privatization, part 3

2017-03-23 Thread Jakub Jelinek 
On Thu, Mar 23, 2017 at 02:13:45PM +0300, Alexander Monakov wrote:
> On Thu, 23 Mar 2017, Jakub Jelinek wrote:
> > On Wed, Mar 22, 2017 at 06:46:34PM +0300, Alexander Monakov wrote:
> > > @@ -4730,6 +4746,25 @@ expand_call_inline (basic_block bb, gimple *stmt, 
> > > copy_body_data *id)
> > >if (cfun->gimple_df)
> > >  pt_solution_reset (>gimple_df->escaped);
> > >  
> > > +  /* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments.  */
> > > +  if (id->dst_simt_vars)
> > > +{
> > > +  size_t nargs = gimple_call_num_args (simtenter_stmt);
> > > +  hash_set *vars = id->dst_simt_vars;
> > > +  auto_vec newargs (nargs + vars->elements ());
> > > +  for (size_t i = 0; i < nargs; i++)
> > > + newargs.quick_push (gimple_call_arg (simtenter_stmt, i));
> > > +  for (hash_set::iterator i = vars->begin (); i != vars->end 
> > > (); ++i)
> > > + newargs.quick_push (build1 (ADDR_EXPR,
> > > + build_pointer_type (TREE_TYPE (*i)), *i));
> > 
> > Traversing a hash table where the traversal affects code generation is
> > -fcompare-debug unfriendly.
> > Do you actually need a hash_set and not say just a vec of the vars?  I can't
> > find where you'd actually do any lookups there, just add and traverse.
> 
> Sorry for missing the IR stability issue.  This code relies on dst_simt_vars
> being a set and thus having no duplicate entries (so the implicit lookup when
> adding an element is needed).
> 
> However, I think I was overly cautious: looking again, I think we can't enter
> copy_decl_for_dup_finish twice with the same 'copy' VAR_DECL.  Changing it to 
> a
> vec should be fine then?

Yeah, callers of copy_decl* should look the orig var in the decl map first,
plus if you look at all copy_decl_for_dup_finish callers, all of them first
create a new tree (usually copy_node) and then pass it as copy to
copy_decl_for_dup_finish, so you'll never get the same copy in there.

So, please change it into a vector.

Jakub

Re: [PATCH 4/5] tree-inline: implement SIMT privatization, part 3

2017-03-23 Thread Alexander Monakov 
On Thu, 23 Mar 2017, Jakub Jelinek wrote:
> On Wed, Mar 22, 2017 at 06:46:34PM +0300, Alexander Monakov wrote:
> > @@ -4730,6 +4746,25 @@ expand_call_inline (basic_block bb, gimple *stmt, 
> > copy_body_data *id)
> >if (cfun->gimple_df)
> >  pt_solution_reset (>gimple_df->escaped);
> >  
> > +  /* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments.  */
> > +  if (id->dst_simt_vars)
> > +{
> > +  size_t nargs = gimple_call_num_args (simtenter_stmt);
> > +  hash_set *vars = id->dst_simt_vars;
> > +  auto_vec newargs (nargs + vars->elements ());
> > +  for (size_t i = 0; i < nargs; i++)
> > +   newargs.quick_push (gimple_call_arg (simtenter_stmt, i));
> > +  for (hash_set::iterator i = vars->begin (); i != vars->end (); 
> > ++i)
> > +   newargs.quick_push (build1 (ADDR_EXPR,
> > +   build_pointer_type (TREE_TYPE (*i)), *i));
> 
> Traversing a hash table where the traversal affects code generation is
> -fcompare-debug unfriendly.
> Do you actually need a hash_set and not say just a vec of the vars?  I can't
> find where you'd actually do any lookups there, just add and traverse.

Sorry for missing the IR stability issue.  This code relies on dst_simt_vars
being a set and thus having no duplicate entries (so the implicit lookup when
adding an element is needed).

However, I think I was overly cautious: looking again, I think we can't enter
copy_decl_for_dup_finish twice with the same 'copy' VAR_DECL.  Changing it to a
vec should be fine then?

Thanks.
Alexander

Re: [PATCH 4/5] tree-inline: implement SIMT privatization, part 3

2017-03-23 Thread Jakub Jelinek 
On Wed, Mar 22, 2017 at 06:46:34PM +0300, Alexander Monakov wrote:
> @@ -4730,6 +4746,25 @@ expand_call_inline (basic_block bb, gimple *stmt, 
> copy_body_data *id)
>if (cfun->gimple_df)
>  pt_solution_reset (>gimple_df->escaped);
>  
> +  /* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments.  */
> +  if (id->dst_simt_vars)
> +{
> +  size_t nargs = gimple_call_num_args (simtenter_stmt);
> +  hash_set *vars = id->dst_simt_vars;
> +  auto_vec newargs (nargs + vars->elements ());
> +  for (size_t i = 0; i < nargs; i++)
> + newargs.quick_push (gimple_call_arg (simtenter_stmt, i));
> +  for (hash_set::iterator i = vars->begin (); i != vars->end (); 
> ++i)
> + newargs.quick_push (build1 (ADDR_EXPR,
> + build_pointer_type (TREE_TYPE (*i)), *i));

Traversing a hash table where the traversal affects code generation is
-fcompare-debug unfriendly.
Do you actually need a hash_set and not say just a vec of the vars?  I can't
find where you'd actually do any lookups there, just add and traverse.

Jakub

[PATCH 4/5] tree-inline: implement SIMT privatization, part 3

2017-03-22 Thread Alexander Monakov 
This patch implements privatization for SIMT during inlining.  We need to
discover if the call being inlined belongs to a SIMT region (by looking at
simduid of the containing loop), and if so, treat them similar to OpenMP-SIMD
privatization: add the "omp simt private" attribute and mention them among
arguments of GOMP_SIMT_ENTER.

OpenMP-SIMD privatization also adds a clobber at the end of the region; I'm
not sure if it's required here: in the example I've looked at, inlined code
already contained a clobber.

* tree-inline.h (struct copy_body_data): New field dst_simt_vars.
* tree-inline.c (expand_call_inline): Handle SIMT privatization.
(copy_decl_for_dup_finish): Ditto.
---
 gcc/tree-inline.c | 59 ---
 gcc/tree-inline.h |  4 
 2 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index 6b6d489..56817e4 100644
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -4385,6 +4385,11 @@ expand_call_inline (basic_block bb, gimple *stmt, 
copy_body_data *id)
   gcall *call_stmt;
   unsigned int i;
   unsigned int prop_mask, src_properties;
+  struct function *dst_cfun;
+  tree simduid;
+  use_operand_p use;
+  gimple *simtenter_stmt = NULL;
+  hash_set *simtvars_st = NULL;
 
   /* The gimplifier uses input_location in too many places, such as
  internal_get_tmp_var ().  */
@@ -4588,15 +4593,26 @@ expand_call_inline (basic_block bb, gimple *stmt, 
copy_body_data *id)
   id->src_cfun = DECL_STRUCT_FUNCTION (fn);
   id->call_stmt = call_stmt;
 
+  /* When inlining into an OpenMP SIMD-on-SIMT loop, arrange for new automatic
+ variables to be added to IFN_GOMP_SIMT_ENTER argument list.  */
+  dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
+  if (!(dst_cfun->curr_properties & PROP_gimple_lomp_dev)
+  && (simduid = bb->loop_father->simduid) != NULL_TREE
+  && (simduid = ssa_default_def (dst_cfun, simduid)) != NULL_TREE
+  && single_imm_use (simduid, , _stmt)
+  && is_gimple_call (simtenter_stmt)
+  && gimple_call_internal_p (simtenter_stmt, IFN_GOMP_SIMT_ENTER))
+{
+  simtvars_st = id->dst_simt_vars;
+  id->dst_simt_vars = new hash_set;
+}
+
   /* If the src function contains an IFN_VA_ARG, then so will the dst
  function after inlining.  Likewise for IFN_GOMP_USE_SIMT.  */
   prop_mask = PROP_gimple_lva | PROP_gimple_lomp_dev;
   src_properties = id->src_cfun->curr_properties & prop_mask;
   if (src_properties != prop_mask)
-{
-  struct function *dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
-  dst_cfun->curr_properties &= src_properties | ~prop_mask;
-}
+dst_cfun->curr_properties &= src_properties | ~prop_mask;
 
   gcc_assert (!id->src_cfun->after_inlining);
 
@@ -4730,6 +4746,25 @@ expand_call_inline (basic_block bb, gimple *stmt, 
copy_body_data *id)
   if (cfun->gimple_df)
 pt_solution_reset (>gimple_df->escaped);
 
+  /* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments.  */
+  if (id->dst_simt_vars)
+{
+  size_t nargs = gimple_call_num_args (simtenter_stmt);
+  hash_set *vars = id->dst_simt_vars;
+  auto_vec newargs (nargs + vars->elements ());
+  for (size_t i = 0; i < nargs; i++)
+   newargs.quick_push (gimple_call_arg (simtenter_stmt, i));
+  for (hash_set::iterator i = vars->begin (); i != vars->end (); ++i)
+   newargs.quick_push (build1 (ADDR_EXPR,
+   build_pointer_type (TREE_TYPE (*i)), *i));
+  gcall *g = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, newargs);
+  gimple_call_set_lhs (g, gimple_call_lhs (simtenter_stmt));
+  gimple_stmt_iterator gsi = gsi_for_stmt (simtenter_stmt);
+  gsi_replace (, g, false);
+  delete id->dst_simt_vars;
+  id->dst_simt_vars = simtvars_st;
+}
+
   /* Clean up.  */
   if (id->debug_map)
 {
@@ -5453,9 +5488,19 @@ copy_decl_for_dup_finish (copy_body_data *id, tree decl, 
tree copy)
function.  */
 ;
   else
-/* Ordinary automatic local variables are now in the scope of the
-   new function.  */
-DECL_CONTEXT (copy) = id->dst_fn;
+{
+  /* Ordinary automatic local variables are now in the scope of the
+new function.  */
+  DECL_CONTEXT (copy) = id->dst_fn;
+  if (VAR_P (copy) && id->dst_simt_vars && !is_gimple_reg (copy))
+   {
+ if (!lookup_attribute ("omp simt private", DECL_ATTRIBUTES (copy)))
+   DECL_ATTRIBUTES (copy)
+ = tree_cons (get_identifier ("omp simt private"), NULL,
+  DECL_ATTRIBUTES (copy));
+ id->dst_simt_vars->add (copy);
+   }
+}
 
   return copy;
 }
diff --git a/gcc/tree-inline.h b/gcc/tree-inline.h
index 88b3286..cf46fa5 100644
--- a/gcc/tree-inline.h
+++ b/gcc/tree-inline.h
@@ -145,6 +145,10 @@ struct copy_body_data
  equivalents in the function into which it is being inlined.  */
   hash_map