osprey/be/com/wn_lower.cxx

Go to the documentation of this file.

/*
 *  Copyright (C) 2006. QLogic Corporation. All Rights Reserved.
 */

/*
 * Copyright 2002, 2003, 2004, 2005, 2006 PathScale, Inc.  All Rights Reserved.
 */

/*

  Copyright (C) 2000, 2001 Silicon Graphics, Inc.  All Rights Reserved.

  This program is free software; you can redistribute it and/or modify it
  under the terms of version 2 of the GNU General Public License as
  published by the Free Software Foundation.

  This program is distributed in the hope that it would be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  

  Further, this software is distributed without any warranty that it is
  free of the rightful claim of any third person regarding infringement 
  or the like.  Any license provided herein, whether implied or 
  otherwise, applies only to this software file.  Patent licenses, if 
  any, provided herein do not apply to combinations of this program with 
  other software, or any other product whatsoever.  

  You should have received a copy of the GNU General Public License along
  with this program; if not, write the Free Software Foundation, Inc., 59
  Temple Place - Suite 330, Boston MA 02111-1307, USA.

  Contact information:  Silicon Graphics, Inc., 1600 Amphitheatre Pky,
  Mountain View, CA 94043, or:

  http://www.sgi.com

  For further information regarding this notice, see:

  http://oss.sgi.com/projects/GenInfo/NoticeExplan

*/


//-*-c++-*-

#define __STDC_LIMIT_MACROS
#include <stdint.h>
#ifdef USE_PCH
#include "be_com_pch.h"
#endif /* USE_PCH */
#pragma hdrstop
#if defined(BUILD_OS_DARWIN)
#include <limits.h>
#else /* defined(BUILD_OS_DARWIN) */
#include <values.h>
#endif /* defined(BUILD_OS_DARWIN) */
#include <isam.h>
#include <alloca.h>
#ifdef __MINGW32__
#include <signal.h>
#else
#include <sys/signal.h>
#endif /* __MINGW32__ */
#if defined(BUILD_OS_DARWIN)
#include <darwin_elf.h>
#else /* defined(BUILD_OS_DARWIN) */
#include <elf.h>
#endif /* defined(BUILD_OS_DARWIN) */

#include "defs.h"
#include "config.h"
#include "config_asm.h"
#include "config_debug.h"
#include "config_opt.h"
#include "config_targ_opt.h"
#include "errors.h"
#include "erglob.h"
#include "tracing.h"
#include "glob.h"
#include "timing.h"
#include "stab.h"
#include "strtab.h"
#include "util.h"
#include "wn.h"
#include "wn_util.h"
#include "stblock.h"
#include "data_layout.h"
#include "ir_reader.h"
#include "targ_sim.h"
#include "targ_const.h"
#include "const.h"
#include "ttype.h"
#include "wio.h"
#include "wn_mp.h"
#include "wn_pragmas.h"
#include "wn_simp.h"
#include "opt_alias_interface.h"
#include "wn_lower.h"
#include "region_util.h"
#include "wutil.h"
#include "wn_map.h"
#include "wn_fio.h"
#include "wn_trap.h"
#include "pu_info.h"
#include "w2op.h"
#include "be_symtab.h"
#include "betarget.h"
#include "be_util.h"
#include "opt_cvtl_rule.h"
#include "fb_whirl.h"
#include "intrn_info.h"
#ifdef KEY
#include "config_vho.h"   // for VHO_Enable_Simple_If_Conv
#include "targ_const_private.h" // for TCON_R4, TCON_R8, ..
#endif
#include "be_memop_annot.h"

#ifdef TARG_X8664
#include <ext/hash_set>
#endif

#include "tls.h"

#ifdef TARG_NVISA
#define SHORTCIRCUIT_TO_LOGICAL_LIMIT 3
#endif

/* My changes are a hack till blessed by Steve. (suneel) */
#define SHORTCIRCUIT_HACK 1
#define MMCOPY_MULT 1.2 
#define MEMLIB_THRESHOLD_BYTES OPT_Threshold_To_Memlib  //Parameter to adjust
#define MEMLIB_STRUCT_BYTES 128  //Parameter to adjust
#define MEMLIB_NUM_STORE 5  //Parameter to adjust
#define MEMLIB_MAX_NUM 6
#ifdef KEY
#define DELETED_MAX 100 
#else
#define DELETED_MAX 20 
#endif

/* this next header should be after the external declarations in the others */
#include "pragma_weak.h"  /* Alias routines defined in wopt.so */

/* ====================================================================
 *       Exported Functions
 * ====================================================================
 */
extern PREG_NUM AssignExpr(WN *, WN *, TYPE_ID);

extern BOOL lower_is_aliased(WN *, WN *, INT64 size, BOOL defalt);

extern INT32 compute_copy_alignment(TY_IDX, TY_IDX, INT32 offset);

extern TYPE_ID compute_copy_quantum(INT32 );

extern WN *WN_I1const(TYPE_ID, INT64 con);

extern void Set_addr_saved_expr(WN *wn, BOOL warn);
/*
 * defined in config.c (should be in config.h)
 */
extern INT32  MinStructCopyLoopSize;
extern INT32  MinStructCopyMemIntrSize;

INT32 Total_MLower_Count=0;
INT32 Total_M_Count=0;

/* ====================================================================
 *       Imported Declarations
 * ====================================================================
 */
extern WN *emulate(WN *, WN *);

#ifdef KEY // bug 6938
extern WN *emulate_fast_exp(WN *, WN *);
#endif

extern WN *intrinsic_runtime(WN *, WN *);

extern WN *make_pointer_to_node(WN *, WN *);

extern char *INTR_intrinsic_name( WN *);

extern void fdump_dep_tree(FILE *, WN *, struct ALIAS_MANAGER *);

extern void InitParityMaps(void);

extern "C" void LNOPreserveMapPair(WN *, WN *, WN *);

extern "C" void LNOPruneMapsUsingParity(void);

extern "C" void LNOPrintDepGraph(FILE *);

extern void enable_tree_freq_display(void);

extern TYPE_ID INTR_return_mtype(INTRINSIC id);

extern BE_ST_TAB   Be_st_tab;

#ifdef TARG_X8664
// Nesting depth is insanely high (120 or so) for bug 599
#define MAX_LOOP_NEST_DEPTH 150
/* <loop_info_stack> has its own <current_loop_nest_depth>, since <loop_nest_depth>
   will be set to 0 by WN_loop_depth(). Most of the time, WN_loop_depth will
   give non-zero value inside a do_loop, but IPA could make it be zero, which could
   cause some other problems later.
   ToDo: Look into IPA to find out what is going on.
*/
static UINT8 current_loop_nest_depth;
static WN* loop_info_stack[MAX_LOOP_NEST_DEPTH];
static BOOL last_call_ff2c_abi; // whether the last outgoing call use this abi
#endif

/* ====================================================================
 *       Forward Declarations
 * ====================================================================
 */
#ifdef LOW_LANDING_PAD
#undef LOW_LANDING_PAD
#endif
#ifdef TARG_X8664
#define LOW_LANDING_PAD
#endif
#ifdef TARG_IA64
#define LOW_LANDING_PAD
#endif

static WN *lower_scf(WN *, WN *, LOWER_ACTIONS);
static WN *lower_expr(WN *, WN *, LOWER_ACTIONS);
static WN *lower_store(WN *, WN *, LOWER_ACTIONS);
static WN *lower_call(WN *, WN *, LOWER_ACTIONS);
static WN *lower_intrinsic(WN *, WN *, LOWER_ACTIONS);
static WN *lower_intrinsic_call(WN *, WN *, LOWER_ACTIONS);
static WN *lower_intrinsic_op(WN *, WN *, LOWER_ACTIONS);
static WN *lower_if(WN *, WN *, LOWER_ACTIONS);
static WN *lower_stmt(WN *, WN *, LOWER_ACTIONS);
static WN *lower_entry(WN *, LOWER_ACTIONS);
#ifdef LOW_LANDING_PAD
static WN *lower_landing_pad_entry(WN *);
#endif 
static WN *lower_eval(WN *, WN *, LOWER_ACTIONS);
static WN *lower_copy_tree(WN *, LOWER_ACTIONS);
static WN *lower_emulation(WN *, WN *, LOWER_ACTIONS, BOOL &intrinsic_lowered);
static WN *lower_mstore(WN *, WN *, LOWER_ACTIONS);
static WN *lower_nary_madd(WN *, WN *, LOWER_ACTIONS);
static WN *lower_madd(WN *, WN *, LOWER_ACTIONS);
static WN *lower_assert(WN *, WN *, LOWER_ACTIONS);
static WN *lower_trapuv(WN *, WN *, LOWER_ACTIONS);
static WN *lower_base_reference(WN *, ST *, INT64, LOWER_ACTIONS);
static WN *lower_base_register(WN *, ST *, INT64, LOWER_ACTIONS);
static WN *lower_dereference(WN *, INT64, ST *, PREG_NUM, LOWER_ACTIONS);
static WN *lower_split_sym_addrs(WN *, INT64, LOWER_ACTIONS);
static WN *improve_Malignment(WN *, WN *, WN *, INT64);
static WN *lower_branch(WN *, WN *, LOWER_ACTIONS);
static WN *lower_branch_condition(BOOL, LABEL_IDX, WN *, WN **, LOWER_ACTIONS);
static WN *lower_conditional(WN *, WN *, LABEL_IDX, LABEL_IDX, BOOL,
           LOWER_ACTIONS);
static WN *lower_tree_height(WN *, WN *, LOWER_ACTIONS);
static void lower_madd_tree_height(WN *, WN *, LOWER_ACTIONS);

static WN *Lower_Memlib(WN *block, WN *tree, LOWER_ACTIONS actions);
static WN *Lower_Mistore_Memlib(WN *block, WN *tree, LOWER_ACTIONS actions);
static WN *Lower_STID_Memlib(WN *block, WN *tree, LOWER_ACTIONS actions);

static TY_IDX coerceTY(TY_IDX, TYPE_ID);
static ST *coerceST(const ST *, TYPE_ID);
static ST *coerceST(const ST &, TYPE_ID);
static WN_OFFSET coerceOFFSET(WN *, TYPE_ID, WN_OFFSET);


static WN *lower_mload(WN *, WN *, LOWER_ACTIONS);
static void lower_mload_formal(WN *, WN *, PLOC, LOWER_ACTIONS);
static void lower_mload_actual (WN *, WN *, PLOC, LOWER_ACTIONS);
static void lower_complex_emulation(WN *, WN *, LOWER_ACTIONS, WN **, WN **);
static void lower_complex_expr(WN *, WN *, LOWER_ACTIONS, WN **, WN **);


static void lower_copy_maps(WN *, WN *, LOWER_ACTIONS);
static void lower_tree_copy_maps(WN *, WN *, LOWER_ACTIONS);

static INT32 compute_alignment(WN *, INT64);
static TYPE_ID compute_next_copy_quantum(TYPE_ID , INT32);

static WN *lower_mistore_by_type(WN *block, WN *mistore, LOWER_ACTIONS actions);
#if defined(KEY) 
static WN *lower_malloc_alg(WN *block, WN *tree, LOWER_ACTIONS actions);
static ST_IDX find_trampoline(ST_IDX func_st_idx);
#endif

// return the WN* from which <derived> was derived
static WN* get_original_wn (WN* derived);
static void set_original_wn (WN* derived, WN* orig);

/* ====================================================================
 *       private variables
 * ====================================================================
 */
static INT   max_region;
static char *current_preg_name;
static UINT16 loop_nest_depth;
static BOOL contains_a_loop;
static struct ALIAS_MANAGER *alias_manager;
#define NESTED_MAP_ARRAY_SIZE 32
static WN_MAP parity_map_array[NESTED_MAP_ARRAY_SIZE];
static WN_MAP derivation_map_array[NESTED_MAP_ARRAY_SIZE];
static WN_MAP lowering_parity_map = WN_MAP_UNDEFINED;
static WN_MAP wn_derivation_map = WN_MAP_UNDEFINED; 
static INT32 nested_map_index = -1;
static LOWER_ACTIONS lowering_actions= 0;
static BOOL save_Div_Split_Allowed ;
static INT cur_memlib_idx        = 0;

// for promote ldid/lda in param in call to up level
static BOOL promote_tls_ldst     = FALSE;
static BOOL traceIO              = FALSE;
static BOOL traceSpeculate       = FALSE;
static BOOL traceAlignment       = FALSE;
static BOOL traceTreeHeight      = FALSE;
static BOOL traceSplitSymOff     = FALSE;
static BOOL traceWoptFinishedOpt = FALSE;
static BOOL traceMload           = FALSE;
static BOOL traceMemlib          = FALSE;

// static BOOL traceUplevel = FALSE;

typedef struct CURRENT_STATE
{
  SRCPOS  srcpos;
  WN    *stmt;
  WN    *function;
  LOWER_ACTIONS actions;
} CURRENT_STATE, *CURRENT_STATEp;

CURRENT_STATE current_state;

#define current_srcpos    current_state.srcpos
#define current_stmt    current_state.stmt
#define current_actions   current_state.actions
#define current_function  current_state.function

typedef enum MSTORE_ACTIONS
{
  MSTORE_aggregate,
  MSTORE_loop,
  MSTORE_intrinsic_bzero,
  MSTORE_intrinsic_memset,
#ifdef KEY
  MSTORE_intrinsic_memcpy,
#endif
  MSTORE_intrinsic_bcopy
} MSTORE_ACTIONS;

enum MEMLIB_ACTIONS
{
  IGNORE,
  ABORT_OPT,
  CONT_OPT
};

typedef struct inode {
   WN *wn;
   INT start_offset;
   INT length;
   struct inode *next;
} inode_type;

static const char * MSTORE_ACTIONS_name(MSTORE_ACTIONS);

static TYPE_ID Promoted_Mtype[MTYPE_LAST + 1] = {
  MTYPE_UNKNOWN,  /* MTYPE_UNKNOWN */
  MTYPE_UNKNOWN,  /* MTYPE_B */
  MTYPE_I4,       /* MTYPE_I1 */
  MTYPE_I4,       /* MTYPE_I2 */
  MTYPE_I4,       /* MTYPE_I4 */
  MTYPE_I8,       /* MTYPE_I8 */
  MTYPE_U4,       /* MTYPE_U1 */
  MTYPE_U4,       /* MTYPE_U2 */
  MTYPE_U4,       /* MTYPE_U4 */
  MTYPE_U8,       /* MTYPE_U8 */
  MTYPE_F4,       /* MTYPE_F4 */
  MTYPE_F8,       /* MTYPE_F8 */
#ifdef TARG_IA64
  MTYPE_F10,      /* MTYPE_F10 */
#else
  MTYPE_UNKNOWN,  /* MTYPE_F10 */
#endif
  MTYPE_UNKNOWN,  /* MTYPE_F16 */
  MTYPE_UNKNOWN,  /* MTYPE_STR */
  MTYPE_FQ,       /* MTYPE_FQ */
  MTYPE_M,        /* MTYPE_M */
  MTYPE_C4,       /* MTYPE_C4 */
  MTYPE_C8,       /* MTYPE_C8 */
  MTYPE_CQ,       /* MTYPE_CQ */
  MTYPE_V         /* MTYPE_V */
#ifdef KEY
 ,MTYPE_UNKNOWN,  /* MTYPE_BS */
  MTYPE_UNKNOWN,  /* MTYPE_A4 */
  MTYPE_UNKNOWN,  /* MTYPE_A8 */
#ifdef TARG_IA64
  MTYPE_C10,    /* MTYPE_C10 */
#else
  MTYPE_UNKNOWN,  /* MTYPE_C10 */
#endif
  MTYPE_UNKNOWN,  /* MTYPE_C16 */
  MTYPE_UNKNOWN,  /* MTYPE_I16 */
  MTYPE_UNKNOWN,  /* MTYPE_U16 */
#if defined(TARG_X8664) || defined(VECTOR_MTYPES)
  MTYPE_V16C4,    /* MTYPE_V16C4 */
  MTYPE_V16C8,    /* MTYPE_V16C8 */
  MTYPE_V16I1,    /* MTYPE_V16I1 */
  MTYPE_V16I2,    /* MTYPE_V16I2 */
  MTYPE_V16I4,    /* MTYPE_V16I4 */
  MTYPE_V16I8,    /* MTYPE_V16I8 */
  MTYPE_V16F4,    /* MTYPE_V16F4 */
  MTYPE_V16F8,    /* MTYPE_V16F8 */
  MTYPE_V8I1,     /* MTYPE_V8I1 */
  MTYPE_V8I2,     /* MTYPE_V8I2 */
  MTYPE_V8I4,     /* MTYPE_V8I4 */
  MTYPE_V8F4,     /* MTYPE_V8F4 */
#if defined(TARG_X8664)
  MTYPE_M8I1,     /* MTYPE_M8I1 */
  MTYPE_M8I2,     /* MTYPE_M8I2 */
  MTYPE_M8I4,     /* MTYPE_M8I4 */
  MTYPE_M8F4      /* MTYPE_M8F4 */
#endif // TARG_X8664
#endif // (TARG_X8664) || (VECTOR_MTYPES)
#endif // KEY
};

/* ====================================================================
 *      Private macros
 * ====================================================================
 */

#define OPCODE_is_intrinsic(op)                                   \
    ((OPCODE_operator((op)) == OPR_INTRINSIC_CALL) ||       \
    (OPCODE_operator((op)) == OPR_INTRINSIC_OP))

#define Action(x)     (actions & (x))
#define NotAction(x)      (Action(x)==0)
#define RemoveScfAction(x)    (x & ~(LOWER_SCF))
#define RemoveShortCircuitAction(x) (x & ~(LOWER_SHORTCIRCUIT))


#define WN_has_alias_info(x)  (OPCODE_is_load(WN_opcode(x)) ||  \
         OPCODE_is_store(WN_opcode(x))  ||  \
         WN_operator_is(x, OPR_PARM))
#define WN_has_offset(x)  (OPCODE_has_offset(WN_opcode(x)))

#define WN_nary_intrinsic(x)  (WN_operator_is(x, OPR_INTRINSIC_OP) && \
              ((WN_intrinsic(x)== INTRN_NARY_ADD) ||  \
               (WN_intrinsic(x)== INTRN_NARY_MPY)))
#define WN_nary_add(x)    (WN_operator_is(x, OPR_INTRINSIC_OP) && \
               (WN_intrinsic(x)== INTRN_NARY_ADD))
#define WN_nary_mpy(x)    (WN_operator_is(x, OPR_INTRINSIC_OP) && \
               (WN_intrinsic(x)== INTRN_NARY_MPY))

#define WN_is_block(x)    (WN_opcode(x) == OPC_BLOCK)

#define INTRN_is_nary(x)  (((x)==INTRN_NARY_ADD) || ((x)==INTRN_NARY_MPY))

#define WN_is_commutative(x)  (WN_opcode(x) == OPCODE_commutative_op(WN_opcode(x)))

#define TY_is_pointer(x)  (TY_kind(x) == KIND_POINTER)

#define lower_truebr(l,c,b,a) lower_branch_condition(TRUE,l,c,b,a)
#define lower_falsebr(l,c,b,a)  lower_branch_condition(FALSE,l,c,b,a)
#define WN_same_id(a,b)         (WN_st(a)==WN_st(b) && WN_offset(a)==WN_offset(b))

/* ====================================================================
 *
 * BOOL mem_offset_must_be_split(WN_OFFSET offset)
 * WN_OFFSET mem_offset_hi(WN_OFFSET offset)
 * WN_OFFSET mem_offset_lo(WN_OFFSET offset)
 *
 * Returns TRUE iff <offset> must be split for a target-machine memory
 * reference.  If so, mem_offset_hi(offset) returns the high part of
 * the split offset, and mem_offset_lo(offset) returns the low part of
 * the split offset.
 * 
 * ==================================================================== */

#define mem_offset_hi(offset) ((offset) & ~0x7fff)
#define mem_offset_lo(offset) ((offset) & 0x7fff)

#define mem_offset_must_be_split(offset)    \
        (!(-32768 <= (offset) && (offset) < 32768))

#define mem_offset_2GB(offset)        \
  (!(INT32_MIN <= offset && offset <= INT32_MAX))

#define PIC_SHARED    (Gen_PIC_Shared || Gen_PIC_Call_Shared)
#define PIC_NONSHARED   (!PIC_SHARED)

#define ABS(x)      (((x)<0) ? -(x) : (x))
#define IS_POWER_OF_2(val)      ((val != 0) && ((val & (val-1)) == 0))

static ST *
Standard_Preg_For_Mtype (TYPE_ID mtype)
{
#ifdef TARG_NVISA
  // need specific-size preg
  if (mtype == MTYPE_F8) 
  return Float64_Preg;
  else if (MTYPE_is_float(mtype)) 
  return Float32_Preg;
  else if (MTYPE_byte_size(mtype) == 8)
  return Int64_Preg;
  else
  return Int32_Preg;
#else
  return MTYPE_is_float(mtype) ? Float_Preg : Int_Preg;
#endif
}

#ifdef KEY
static void WN_copy_linenum (WN* src, WN* dest)
{
  if (src && dest &&
      WN_operator(src) != OPERATOR_UNKNOWN &&
      WN_operator(dest) != OPERATOR_UNKNOWN &&
      OPCODE_is_stmt(WN_opcode(src)) && OPCODE_is_stmt(WN_opcode(dest)))
    WN_linenum(dest) = WN_linenum(src);

  return;
}
#endif
/* ====================================================================
 *
 * UINT32 compute_offset_alignment(INT32 offset, UINT32 align)
 *
 * return gcd of offset,align;
 * Used for alignment reasons;
 * For maximum efficiency, offset should be >= align
 *
 * ==================================================================== */
UINT32 compute_offset_alignment(INT32 offset, UINT32 align)
{
  UINT32 m = ABS(offset);
  UINT32 n = align;
  while (n != 0) {
    INT32 new_n = m % n;
    m = n;
    n = new_n;
  }
  return m;
}


/* ====================================================================
 *
 *  The semantics of memset require replicating the byte constant
 *  so replicate the constant into a I8/U8 depending on WN_rtype(con)
 *
 * ==================================================================== */
extern WN *WN_I1const(TYPE_ID type, INT64 con)
{
  // assume con is a byte constant, so clear the other bits
  // (otherwise replicate will "or" with sign bits).
  INT64 n=    con & 0xff;
  INT64 maxAlign= MTYPE_alignment(Max_Uint_Mtype);

  if (con)
  {
    INT64 i;
    for(i=1; i<maxAlign; i++)
    {
      n |=  (n << 8);
    }
  }

  return WN_Intconst(Mtype_AlignmentClass(maxAlign,
            MTYPE_type_class(type)), n);
}


/* ====================================================================
 *
 * void push_current_state(WN *tree, STATE *state)
 *
 * return and set current state
 *
 * ==================================================================== */
static void setCurrentState(WN *tree, LOWER_ACTIONS actions)
{
  if (tree) {
    current_stmt =  tree;
    current_srcpos =  WN_Get_Linenum(tree);
    current_actions = actions;

    if (WN_opcode(tree) == OPC_FUNC_ENTRY)
      current_function = tree;
  }
}

static void setCurrentStateBlockFirst(WN *tree, LOWER_ACTIONS actions)
{
  Is_True(WN_opcode(tree) == OPC_BLOCK, ("expected BLOCK node"));

  setCurrentState(WN_first(tree), actions);
}

static void setCurrentStateBlockLast(WN *tree, LOWER_ACTIONS actions)
{
  Is_True(WN_opcode(tree) == OPC_BLOCK, ("expected BLOCK node"));

  setCurrentState(WN_last(tree), actions);
}


static CURRENT_STATE pushCurrentState(WN *tree, LOWER_ACTIONS actions)
{
  CURRENT_STATE saveState = current_state;

  setCurrentState(tree, actions);

  return saveState;
}

static void popCurrentState(CURRENT_STATE state)
{
  current_state = state;
}
    

/* ====================================================================
 *
 * BOOL foldConstOffset(WN *con, INT64 offset)
 *
 * BOOL foldLdaOffset(WN *lda, INT64 offset)
 *
 * can con and offset be folded to "fit" into a WN_OFFSET (INT32?)
 *
 * ==================================================================== */

static BOOL foldConstOffset(WN *con, INT64 offset)
{
  if (WN_operator_is(con, OPR_INTCONST))
  {
    INT64 sum= offset + WN_const_val(con);

    if (INT32_MIN <= sum && sum <= INT32_MAX)
      return TRUE;
  }
  return FALSE;
}

static BOOL foldLdaOffset(WN *lda, INT64 offset)
{
  if (WN_operator_is(lda, OPR_LDA))
  {
    INT64 sum= offset + WN_lda_offset(lda);

    Is_True((WN_class(lda) != CLASS_PREG), ("lda class is PREG!!!"));

    if (INT32_MIN <= sum && sum <= INT32_MAX)
    {
      ST  *sym= WN_st(lda);

     /*
      * if the offset is greater than the symbol size, we may generate
      * a relocation that is out of bounds (ex LFTR). See pv 482353 for
      * this rare condition
      */
      if (ST_class(sym) == CLASS_BLOCK && STB_size(sym) < sum)
  return FALSE;

      return TRUE;
    }
  }
  return FALSE;
}








/* ====================================================================
 *
 * WN_OFFSET coerceOFFSET(WN *tree, TYPE_ID realTY, WN_OFFSET offset)
 *
 * The offset may either be an offset or preg number.
 *
 * There is an amazing kludge for complex return values where we
 * return F0, F2
 *
 * ==================================================================== */
static WN_OFFSET coerceOFFSET(WN *tree, TYPE_ID realTY, WN_OFFSET offset)
{

  switch (WN_operator(tree))
  {
  case OPR_ILOAD:
  case OPR_ILOADX:
  case OPR_ISTORE:
  case OPR_ISTOREX:
    return offset + MTYPE_RegisterSize(realTY);

  case OPR_LDID:
  case OPR_STID:
    if (WN_class(tree) == CLASS_PREG)
    {
#ifdef TARG_MIPS
     /*
      *  amazing kludge
      *  for dedicated return register (F0) the ABI defines [F0,F2]
      *  as the return values
      */
      if (Preg_Is_Dedicated(offset) && offset == Float_Preg_Min_Offset)
      {
    return Float_Preg_Min_Offset + 2;
      }
      else
#endif
  return offset + Preg_Increment(realTY);
    }
    return offset + MTYPE_RegisterSize(realTY);
  }
  Fail_FmtAssertion("unexpected complex op (%s)",
        OPCODE_name(WN_opcode(tree)));
  /*NOTREACHED*/
}

/* ====================================================================
 *
 *
 * ==================================================================== */

static void rename_preg(const char *f, const char *preg_class)
{
  static char name[41];

  name[0] = '\0';
  
  strncat(name, f, 30);

  if (preg_class)
  {
    strncat(name, preg_class, 10);
  }
  current_preg_name = name;
}

static void rename_reset(void)
{
  current_preg_name = NULL;
}


static void WN_Set_Flags(WN *src, WN *dst)
{
  if (OPCODE_has_flags(WN_opcode(src)))
  {
    Is_True(OPCODE_has_flags(WN_opcode(dst)), ("expected wn with flags"));
    WN_set_flag(dst, WN_flag(src));
  }
}

extern void WN_annotate_intrinsic_flags(INTRINSIC id, ST *sym)
{
  if (INTRN_is_pure(id)) {
    Set_PU_is_pure(Get_Current_PU());
  }
  if (INTRN_has_no_side_effects(id)) {
    Set_PU_no_side_effects(Get_Current_PU());
  }
}

extern void WN_annotate_call_flags(WN *call, ST *sym)
{
  WN_Set_Call_Default_Flags(call);

  if (PU_no_side_effects(Pu_Table[ST_pu(sym)]))
  {   
    WN_Reset_Call_Non_Data_Mod(call);
    WN_Reset_Call_Non_Parm_Mod(call);
    WN_Reset_Call_Parm_Mod(call);

  }
  if (PU_is_pure(Pu_Table[ST_pu(sym)]))
  {
    WN_Reset_Call_Non_Data_Mod(call);
    WN_Reset_Call_Non_Parm_Mod(call);
    WN_Reset_Call_Parm_Mod(call);

    WN_Reset_Call_Non_Data_Ref(call);
    WN_Reset_Call_Non_Parm_Ref(call);
  }
}

/* ====================================================================
 *
 *
 * Create a new label with linenum info
 *
 * ==================================================================== */
static LABEL_IDX NewLabel(void)
{
  LABEL_IDX label;
  LABEL& lab = New_LABEL(CURRENT_SYMTAB, label);

#ifdef KEY
  /* For IPA's sake, do not generate a specific label name, which will
     conflict with some other basic blocks of other PUs.
  */

  if( Run_ipl ){
    return label;
  }
#endif

  // create label name
  char *name = (char *) alloca (strlen(".L..") + 8 + 8 + 1);
  sprintf(name, "%s%s%d%s%d", LABEL_PREFIX, Label_Name_Separator, 
  Current_PU_Count(), Label_Name_Separator, label);
  LABEL_Init (lab, Save_Str(name), LKIND_DEFAULT);
  return label;
}

static WN *WN_Label(LABEL_IDX l)
{
  WN *label = WN_CreateLabel(ST_IDX_ZERO, l, 0, NULL);
  WN_Set_Linenum(label, current_srcpos);
  return label;
}

static WN *WN_NewLabel(void)
{
  LABEL_IDX label;
  label = NewLabel();
  return WN_Label(label);
}



/* ====================================================================
 *
 *
 * Create false/true branch with line info
 *
 * ==================================================================== */
static WN *WN_Falsebr(LABEL_IDX label, WN *cond)
{
  WN *branch = WN_CreateFalsebr(label, cond);
  WN_Set_Linenum(branch, current_srcpos);
  return branch;
}

static WN *WN_Truebr(LABEL_IDX label, WN *cond)
{
  WN *branch = WN_CreateTruebr(label, cond);
  WN_Set_Linenum(branch, current_srcpos);
  return branch;
}

static WN *WN_Goto(LABEL_IDX label)
{
  WN *branch =  WN_CreateGoto((ST_IDX) 0, label);
  WN_Set_Linenum(branch, current_srcpos);
  return branch;
}

static BOOL expr_is_speculative(WN *tree)
{
  if (OPT_Lower_Speculate)
  {
    BOOL  speculate;

    speculate = WN_Expr_Can_Be_Speculative(tree, alias_manager);
    if (traceSpeculate && speculate)
    {
      DevWarn("WN_lower: found speculative expression: line %d",
        Srcpos_To_Line(current_srcpos));
    }
    return speculate;
  }
  return FALSE;
}

/* ====================================================================
 *
 *
 * Create a Nary representation of the expression passed  
 *
 * ==================================================================== */

static WN *WN_Nary(WN *tree, WN *x0, WN *x1, WN *x2, WN *x3)
{
  TYPE_ID type = WN_rtype(tree);
  OPCODE  op;
  INTRINSIC id;

  op = OPCODE_make_op (OPR_INTRINSIC_OP, type, MTYPE_V);
 /*
  *  figure out id from tree
  */
  switch(WN_operator(tree))
  {
  case OPR_ADD:
  case OPR_SUB:
    id = INTRN_NARY_ADD;
    break;
  case OPR_MPY:
    id = INTRN_NARY_MPY;
    break;
  default:  
    Is_True(FALSE,("unexpected nary op"));
  }

  {
    WN    *args[4], **kids;
    INT16 i, n, argN;

   /*
    *  if the child is a nary op of the same type, integrate it
    */
    n= argN = 0;
    if (x0)
      args[n++] = x0;
    if (x1)
      args[n++] = x1;
    if (x2)
      args[n++] = x2;
    if (x3)
      args[n++] = x3;

    for(i=0; i<n; i++)
    {
      WN *wn = (WN *) args[i];

      if (WN_opcode(wn) == op && WN_intrinsic(wn) == id)
  argN += WN_kid_count(wn);
      else
        argN++;
    }

    kids = (WN **) alloca(argN * sizeof(WN *));

    for(i= argN= 0; i<n; i++)
    {
      WN *wn = args[i];

      if (WN_opcode(wn) == op && WN_intrinsic(wn) == id)
      {
  INT16  j;

  for(j=0; j< WN_kid_count(wn); j++)
    kids[argN++] = WN_kid(wn, j);

  WN_Delete(wn);
      }
      else
      {
  kids[argN++] = wn;
      }
    }

    WN_Delete(tree);
    tree = WN_Create_Intrinsic(op, id, argN, kids);
  }
  return tree;
}





/* ====================================================================
 *
 * WN *WN_ExprToNary(WN *tree, TYPE_ID type)
 *
 * Create a Nary representation of the expression passed  
 *
 * The nary representation looks like an INTRINSIC_OP    
 * with an id = INTRN_NARY_ADD | INTRN_NARY_MPY   
 *
 * ==================================================================== */
extern WN *WN_ExprToNaryType(WN *tree, TYPE_ID type, INT count)
{
  WN  *l, *r;

  if (count > 16) {
  Lmt_DevWarn(1,("WN_ExprToNaryType more than 16 deep, so stop"));
  return tree;
  }
  switch(WN_operator(tree))
  {
  case OPR_ADD:
  case OPR_MPY:
    l = WN_kid0(tree);
    r = WN_kid1(tree);

    if ((WN_opcode(tree) == WN_opcode(l)) &&
        (WN_opcode(tree) == WN_opcode(r)))
    {
      WN_kid0(l) = WN_ExprToNaryType(WN_kid0(l), type, count+1);
      WN_kid1(l) = WN_ExprToNaryType(WN_kid1(l), type, count+1);
      WN_kid0(r) = WN_ExprToNaryType(WN_kid0(r), type, count+1);
      WN_kid1(r) = WN_ExprToNaryType(WN_kid1(r), type, count+1);

      tree = WN_Nary(tree, WN_kid0(l), WN_kid1(l), WN_kid0(r), WN_kid1(r));
      WN_Delete(l);
      WN_Delete(r);
    }
    else if (WN_opcode(tree) == WN_opcode(l))
    {
      WN_kid0(l) = WN_ExprToNaryType(WN_kid0(l), type, count+1);
      WN_kid1(l) = WN_ExprToNaryType(WN_kid1(l), type, count+1);

      tree = WN_Nary(tree, WN_kid0(l), WN_kid1(l), r, NULL);
      WN_Delete(l);
    }
    else if (WN_opcode(tree) == WN_opcode(r))
    {
      WN_kid0(r) = WN_ExprToNaryType(WN_kid0(r), type, count+1);
      WN_kid1(r) = WN_ExprToNaryType(WN_kid1(r), type, count+1);

      tree = WN_Nary(tree, WN_kid0(r), WN_kid1(r), l, NULL);
      WN_Delete(r);
    }
    break;

  case OPR_SUB:
    {
      l = WN_kid0(tree);
      r = WN_kid1(tree);

      if (WN_operator_is(l, OPR_ADD))
      {
  TYPE_ID rtype = WN_rtype(tree);

  WN  *neg = WN_Neg(rtype, r);

  WN_kid0(l) = WN_ExprToNaryType(WN_kid0(l), type, count+1);
  WN_kid1(l) = WN_ExprToNaryType(WN_kid1(l), type, count+1);

  tree = WN_Nary(tree, WN_kid0(l), WN_kid1(l), neg, NULL);
  WN_Delete(l);
      }
    }
    break;
  }
  return tree;
}



/* ====================================================================
 *
 * WN *WN_NaryToExpr(WN *tree)
 *
 * The nary representation looks like an INTRINSIC_OP    
 * with an id = INTRN_NARY_ADD | INTRN_NARY_MPY   
 *
 * ==================================================================== */
extern WN *WN_NaryToExpr(WN *tree)
{
  if (WN_nary_intrinsic(tree))
  {
    INT16 i;
    WN    *wn = WN_kid0(tree);
    INTRINSIC id = (INTRINSIC) WN_intrinsic(tree);
    TYPE_ID rtype = WN_rtype(tree);
    INT   num_parms = WN_kid_count(tree);

    for (i = 1; i < num_parms; i++)
    {
      WN *actual = WN_kid(tree, i);

      actual = WN_NaryToExpr(actual);

      switch(id)
      {
      case INTRN_NARY_ADD:
  if (WN_operator_is(actual, OPR_NEG))
  {
    wn = WN_Sub(rtype, wn, WN_kid0(actual));
    WN_Delete(actual);
  }
  else
  {
    wn = WN_Add(rtype, wn, actual);
  }
        break;
      case INTRN_NARY_MPY:
  wn = WN_Mpy(rtype, wn, actual);
        break;
      }
    }
    return wn;
  }

  return tree;
}



/* ====================================================================
 *
 * WN *WN_NaryDelete(WN *tree, INT32 n)
 *
 * Delete the nth kid of a nary intrinsic op
 * The rest of the children get moved and the num_kids are updated
 *
 * ==================================================================== */

static WN *WN_NaryDelete(WN *tree, INT32 n)
{
  INT32 i;
  INT32 num_kids= WN_kid_count(tree);

  Is_True((n<num_kids),("cannot delete nth kid"));
  Is_True(WN_nary_intrinsic(tree),("expected nary op"));

  for(i=n+1; i<num_kids; i++)
  {
    WN_actual(tree, i-1) = WN_actual(tree, i);
  }
  WN_set_kid_count(tree, WN_kid_count(tree)-1);

  return tree;
}

/* ====================================================================
 *
 * Get offset field (avoid preg offsets)
 * amazing I cannot find anything like this
 *
 * ==================================================================== */

extern INT64 lower_offset(WN *tree, INT64 offset)
{
  if (WN_has_offset(tree))
  {
    switch(WN_operator(tree))
    {
    case OPR_LDA:
      offset += WN_lda_offset(tree);
      break;
    case OPR_MSTORE:
    case OPR_ISTORE:
    case OPR_ISTOREX:
      offset += WN_store_offset(tree);
      break;
    case OPR_STID:
      if (WN_class(tree) != CLASS_PREG)
        offset += WN_store_offset(tree);
      break;
    case OPR_LDID:
      if (WN_class(tree) != CLASS_PREG)
  offset += WN_load_offset(tree);
      break;
    case OPR_MLOAD:
    case OPR_ILOAD:
    case OPR_ILOADX:
      offset += WN_load_offset(tree);
      break;
    }
  }
  return offset;
}


/* ====================================================================
 *
 * Compute alignment consistent with address and offset
 *
 * The new alignment may be improved (or not. see pv [559228])
 *
 * ==================================================================== */
extern TY_IDX compute_alignment_type(WN *tree, TY_IDX type, INT64 offset)
{
  INT32 newAlign;

  newAlign= compute_alignment(tree, lower_offset(tree, 0));

  newAlign= compute_offset_alignment(offset, newAlign);

  if (TY_align(type) != newAlign)
    Set_TY_align(type, newAlign);

  return type;
}

static INT32 compute_alignment(WN *tree, INT64 offset)
{
  WN    *addr;
  TY_IDX   type;
  INT32    align, align0, align1;

  switch(WN_operator(tree))
  {
  case OPR_MSTORE:
    type = TY_pointed(Ty_Table[WN_ty(tree)]);
    addr = WN_kid1(tree);

    if (WN_has_sym(addr) && WN_ty(addr))
    {
      return compute_alignment(addr, offset+WN_lda_offset(addr));
    }
    align = TY_align(type);
    break;

  case OPR_MLOAD:
    type = TY_pointed(Ty_Table[WN_ty(tree)]);
    addr = WN_kid0(tree);

    if (WN_has_sym(addr) && WN_ty(addr))
    {
      return compute_alignment(addr, offset+WN_lda_offset(addr));
    }
    align = TY_align(type);
    break;

  case OPR_ILOAD:
  case OPR_ILOADX:
  case OPR_LDA:
  case OPR_LDID:
    type = WN_ty(tree);
    if (TY_is_pointer(Ty_Table[type]))
    {
      type = TY_pointed(Ty_Table[type]);
    }
    else
    {
      return 1;
    }
    align = TY_align(type);
    break;

  case OPR_ARRAY:
    align=  compute_alignment(WN_array_base(tree), offset);
    offset= WN_element_size(tree);
    break;

  case OPR_ADD:
  case OPR_SUB:
    align0= compute_alignment(WN_kid0(tree), 0);
    align1= compute_alignment(WN_kid1(tree), 0);
    align=  MIN(align0, align1);
    break;

  case OPR_INTCONST:
    offset= WN_const_val(tree);
    align=  MTYPE_alignment(Max_Uint_Mtype);
    break;

  default:
    if (traceAlignment)
    {
      DevWarn("compute_alignment(): unrecognized WN returning alignment of 1");
    }
    return 1;
  }

  align=  compute_offset_alignment(offset, MAX(1, align));

  if (WN_has_sym(tree))
  {
    INT32 newAlign = align;
    ST    *sym = WN_st(tree);

    if (WN_operator_is(tree, OPR_LDA))
    {
      newAlign= ST_alignment(sym);
    }
    else if (WN_operator_is(tree, OPR_LDID) &&
       ST_type(sym)     &&
       TY_is_pointer(Ty_Table[ST_type(sym)]))
    {
      newAlign = TY_align( TY_pointed(Ty_Table[ST_type(sym)]));
    }

    align = compute_offset_alignment(offset, MAX(newAlign, align));
  }
  return align;
}


/* ====================================================================
 *
 * BOOL lower_is_aliased(WN *wn1, WN *wn2, INT64 size)
 *
 * Are these addresses aliased? (used in bcopy/memcpy/memmove)
 *
 * ==================================================================== */
extern BOOL lower_is_aliased(WN *wn1, WN *wn2, INT64 size)
{
  if (alias_manager &&
      Valid_alias(alias_manager, wn1) &&
      Valid_alias(alias_manager, wn2) &&
      (Aliased(alias_manager, wn1, wn2) == NOT_ALIASED))
  {
    return FALSE;
  }

  if (WN_operator_is(wn1, OPR_LDA) && WN_operator_is(wn2, OPR_LDA)) {
    ST  *sym1 = WN_st(wn1);
    ST  *sym2 = WN_st(wn2);
    ST  *base1, *base2;
    INT64 newoffset1, newoffset2;

    if (sym1 != sym2) return FALSE;

    Base_Symbol_And_Offset_For_Addressing(
        sym1, WN_lda_offset(wn1), &base1, &newoffset1);
    Base_Symbol_And_Offset_For_Addressing(
        sym2, WN_lda_offset(wn2), &base2, &newoffset2);

    if (ABS(newoffset1 - newoffset2) >= size) return FALSE;
  }

  return TRUE;
}



/* ====================================================================
 *
 * WN *lower_copy_tree(WN *tree, INT32 n)
 *
 * Copy the tree and duplicate the maps
 *
 * ==================================================================== */

static void lower_tree_copy_maps(WN *tree, WN *dup, LOWER_ACTIONS actions)
{
  if (tree == NULL)
  {
    Is_True((dup == NULL),("inconsistency while copying trees"));
  }
  Is_True((WN_opcode(tree) == WN_opcode(dup)),
    ("inconsistency while copying trees"));

  if (WN_has_map_id(tree) && WN_has_alias_info(tree))
  {
    lower_copy_maps(tree, dup, actions);
  }
  
  if (WN_opcode(tree) == OPC_BLOCK)
  {
    WN  *treeStmt = WN_first(tree);
    WN  *dupStmt  = WN_first(dup);

    while(treeStmt)
    {
      lower_tree_copy_maps(treeStmt, dupStmt, actions);
      treeStmt = WN_next(treeStmt);
      dupStmt  = WN_next(dupStmt);
    }
  }
  else
  {
    INT n;
    for(n = 0; n < WN_kid_count(tree); n++)
    {
      if (WN_kid(tree, n))
      {
  lower_tree_copy_maps(WN_kid(tree,n), WN_kid(dup,n), actions);
      }
    }
  }
}

static WN *lower_copy_tree(WN *tree, LOWER_ACTIONS actions)
{
  WN  *dup;

  dup = WN_COPY_Tree(tree);

  lower_tree_copy_maps(tree, dup, actions);

  return dup;
}

/* ====================================================================
 *
 * PREG_NUM AssignPregExprPos(WN *block, WN *tree, TY_IDX ty, SRCPOS srcpos,
 *                            LOWER_ACTIONS actions)
 *
 * PREG_NUM AssignExprPos(WN *block, WN *tree, TYPE_ID type, SRCPOS srcpos,
 *                        LOWER_ACTIONS)
 *
 * PREG_NUM AssignExpr(WN *block, WN *tree, TYPE_ID type)
 *
 * PREG_NUM AssignExprTY(WN *block, WN *tree, TY_IDX ty)
 *
 * Allocate a preg of type ST_type(preg) and assign expression tree to it
 * and attach it to block. 
 *
 * Assign srcpos (if not NULL)
 *
 * ==================================================================== */

static PREG_NUM AssignPregExprPos(WN *block, WN *tree, TY_IDX ty,
          SRCPOS srcpos, LOWER_ACTIONS actions)
{
  PREG_NUM  pregNo;
  TYPE_ID type;
  ST    *preg = MTYPE_To_PREG(TY_mtype(Ty_Table[ty]));

  Is_True((WN_operator_is(tree, OPR_PARM)==FALSE),("bad parm"));

  type = TY_mtype(Ty_Table[ty]);
  pregNo = Create_Preg(type, current_preg_name);
#ifdef TARG_NVISA
  // we need to track what memory is being accessed when storing
  // an lda into a preg.  So if tree has an lda, or indirects to lda
  // through another preg, put that in preg table.
  WN *lda = Find_Lda (tree);
  if (lda) Set_Preg_Lda (pregNo, lda);
#endif

  {
    WN  *stBlock, *stid;

    stid = WN_Stid(type, pregNo, preg, ty, tree);

#ifdef TARG_NVISA
    Set_addr_saved_expr(tree, TRUE);
#endif

    if (srcpos)
      WN_Set_Linenum (stid, srcpos);

    stBlock = WN_CreateBlock();

   /*
    * This lowering may leed to infinite regress if the
    *   children cannot be lowered (and are allocated a temp, for example) 
    */
    if (actions)
      stid = lower_store(stBlock, stid, actions);

    WN_INSERT_BlockLast(stBlock, stid);

    WN_INSERT_BlockLast(block, stBlock);
  }

  return pregNo;
}


extern PREG_NUM AssignExprTY(WN *block, WN *tree, TY_IDX type)
{
  return AssignPregExprPos(block, tree, type, current_srcpos,
         current_actions);
}

extern PREG_NUM AssignExpr(WN *block, WN *tree, TYPE_ID type)
{
  return AssignPregExprPos(block, tree, MTYPE_To_TY(type), current_srcpos,
         current_actions);
}



static BOOL WN_unconditional_goto(WN *tree)
{
  switch(WN_operator(tree))
  {
  case OPR_GOTO:
  case OPR_REGION_EXIT:
  case OPR_RETURN:
  case OPR_RETURN_VAL:
#ifdef KEY
  case OPR_GOTO_OUTER_BLOCK:
#endif
    return TRUE;
  }
  return FALSE;
}




/* ====================================================================
 *
 * PARITY WN_parity(WN *tree)
 *
 * return the PARITY associated with a tree.
 *
 * Parity encapsulates dependence information, like complex real, imag
 * ==================================================================== */

PARITY WN_parity(WN *tree)
{
  if (WN_map_id(tree) != -1)
  {
    INT32 map;

    Is_True((lowering_parity_map != WN_MAP_UNDEFINED), ("parity map not initialized"));
#ifdef KEY
    // Node may be deleted in which case the corresponding vertex and the 
    // associated edges should be deleted in the dependence graph but there
    // are lots of places in wn_lower.cxx that delete WNs without doing the
    // appropriate update to the dependence graph. So, assume the worst here.
    if (WN_operator(tree) == OPERATOR_UNKNOWN) return PARITY_UNKNOWN;
#endif
    map = WN_MAP32_Get(lowering_parity_map, tree);
   
    if (map!=0)
      return (PARITY) map;
  }
  return PARITY_UNKNOWN;
}

BOOL WN_parity_independent(WN *wn1, WN *wn2)
{

  if (wn1 == NULL   ||
      wn2 == NULL   ||
      WN_map_id(wn1) == -1  ||
      WN_map_id(wn2) == -1)
    return FALSE;

  {
    PARITY p1 = WN_parity(wn1);
    PARITY p2 = WN_parity(wn2);

   return (p1 & p2) ? FALSE : TRUE;
  }
}


static WN* get_original_wn (WN* derived) {
  WN* orig = NULL;
  if (wn_derivation_map != WN_MAP_UNDEFINED) {
    orig = (WN*)WN_MAP_Get (wn_derivation_map, derived);
  }
  
  return orig;
}
 
static void set_original_wn (WN* derived, WN* orig) {
  if (wn_derivation_map != WN_MAP_UNDEFINED) {
    WN_MAP_Set (wn_derivation_map, derived, (void*)orig);
  }
}


/* ====================================================================
 *
 *  MAP PRESERVATION
 *
 *
 * void lower_copy_maps(WN *orig, WN *tree, LOWER_ACTIONS action)
 *
 * void lower_complex_maps(WN *orig, WN *real, WN *imag, LOWER_ACTIONS action)
 *
 * void lower_quad_maps(WN *orig, WN *hipart, WN *lopart, LOWER_ACTIONS action)
 *
 *
 *  copy alias information to tree
 *
 * ==================================================================== */

static void lower_maps_init(LOWER_ACTIONS actions)
{

}

static void lower_maps_reset(LOWER_ACTIONS actions)
{
  if (Action(LOWER_DEPGRAPH_MAPS))
  {
    LNOPruneMapsUsingParity();
  }
}

static void lower_map(WN *tree, LOWER_ACTIONS actions)
{
  if (Action(LOWER_ALIAS_MAPS))
  {
    if (alias_manager)
    {
      if (Valid_alias(alias_manager, tree) == FALSE) {
        WN* orig = get_original_wn (tree);
        if (orig && Valid_alias (alias_manager, orig)) {
    Copy_alias_info(alias_manager, orig, tree);
  } else {
    Create_alias(alias_manager, tree);
  }
      }
    }
  }
}

static void lower_copy_maps(WN *orig, WN *tree, LOWER_ACTIONS actions)
{
  if (orig == NULL)
    return;

 /*
  * The tree may no longer be valid at the point of call
  * (ie. may be deleted) so we must check validity
  */
  if (WN_has_map_id(orig))
  {
    if (WN_has_alias_info(orig) &&
        WN_has_alias_info(tree))
    {
      if (Action(LOWER_PREFETCH_MAPS))
      {
        WN_CopyMap(tree, WN_MAP_PREFETCH, orig);
      }
      if (Action(LOWER_ALIAS_MAPS))
      {
        if (alias_manager)
    Copy_alias_info(alias_manager, orig, tree);
      }

      // maintain the annotation whenever possible
      WN_MEMOP_ANNOT_MGR* p = WN_MEMOP_ANNOT_MGR::WN_mem_annot_mgr();
      if (p) {
        p->Copy_annot (orig, tree); 
      }
    }
  }
  WN_Set_Flags(orig, tree);
}


/*
 * If an original node has a TY with an f90_pointer attribute on it, copy it
 * to the TYs of the new nodes. node2 might be NULL (for cases in which we
 * only produce one new node)
 */
static void lower_copy_tys (WN *orig, WN *node1, WN *node2) 
{
  TY_IDX  ty;

  if (WN_operator_is(orig, OPR_ILOAD))
  {
    ty =  WN_load_addr_ty(orig);

    if (TY_is_f90_pointer(Ty_Table[ty]))
    {
      if (node1 && WN_operator_is(node1, OPR_ILOAD))
  WN_set_load_addr_ty(node1, ty);

      if (node2 && WN_operator_is(node2, OPR_ILOAD))
  WN_set_load_addr_ty(node2, ty);
    }
  }
  else if (WN_operator_is(orig, OPR_ISTORE))
  {
    ty =  WN_ty(orig);
    if (TY_is_f90_pointer(Ty_Table[ty]))
    {
      if (node1 && WN_operator_is(node1, OPR_ISTORE))
  WN_set_ty(node1, ty);
      if (node2 && WN_operator_is(node2, OPR_ISTORE))
  WN_set_ty(node2, ty);
    }
  }
}


static void lower_quad_maps(WN *orig, WN *hipart, WN *lopart,
          LOWER_ACTIONS actions)
{
  lower_copy_maps(orig, hipart, actions);
  lower_copy_maps(orig, lopart, actions);
  lower_copy_tys(orig,hipart,lopart);

  if (Action(LOWER_DEPGRAPH_MAPS))
  {
    LNOPreserveMapPair(orig, hipart, lopart);
  }

  if (Action(LOWER_PARITY_MAPS))
  {
    WN_MAP32_Set(lowering_parity_map, hipart, PARITY_QUAD_HI);
    WN_MAP32_Set(lowering_parity_map, lopart, PARITY_QUAD_LO);
  }
}

static void lower_complex_maps(WN *orig, WN *real, WN *imag,
             LOWER_ACTIONS actions)
{
  lower_copy_maps(orig, real, actions);
  lower_copy_maps(orig, imag, actions);
  lower_copy_tys(orig,real,imag);

  if (Action(LOWER_DEPGRAPH_MAPS))
  {
    LNOPreserveMapPair(orig, real, imag);
  }

  if (Action(LOWER_PARITY_MAPS))
  {
    WN_MAP32_Set(lowering_parity_map, real, PARITY_COMPLEX_REAL);
    WN_MAP32_Set(lowering_parity_map, imag, PARITY_COMPLEX_IMAG);
  }

}

static WN *add_to_base(WN **base, WN *tree)
{
  if (*base)
  {
    return WN_Add(Pointer_type, *base, tree);
  }
  return tree;
}

static WN *sub_from_base(WN **base, WN *tree)
{
  if (*base)
  {
    return WN_Sub(Pointer_type, *base, tree);
  }
  return tree;
}

static BOOL baseAddress(WN *tree)
{
  switch(WN_operator(tree))
  {
  case OPR_LDA:
  case OPR_LDID:
  case OPR_ILOAD:
  case OPR_ILOADX:
    return TRUE;
  }
  return FALSE;
}


/* ====================================================================
 *
 * void lower_to_base_index(WN *addr, WN **base, WN **index) 
 *
 * Pattern match an address and create a bad/index for it
 *
 * Before this routine we would store the address in a preg
 * and use it (twice).  This would require the ST being marked
 * addr_taken_stored (very bad)
 *
 * This is getting worse and worse. Now the routine is recursive
 * for OPR_ADD.
 * 
 * 17 Dec 1998, R. Shapiro - Add a default case so that this routine never 
 *   fails. At worst, the trees get a little bigger. 
 *
 * ==================================================================== */

static void lower_to_base_index(WN *addr, WN **base, WN **index) 
{
  WN  *l, *r;

  switch (WN_operator(addr))
  {
  case OPR_ARRAY:
    *base  = add_to_base(base, WN_array_base(addr));
    *index = add_to_base(index, addr);
    WN_array_base(addr) = WN_Zerocon(WN_rtype(addr));
    break;

  case OPR_ADD:
    l = WN_kid0(addr);
    r = WN_kid1(addr);

    if (baseAddress(l))
    {
      *base  = add_to_base(base, l);
      *index = add_to_base(index, r);
    }
    else if (baseAddress(r))
    {
      *base  = add_to_base(base, r);
      *index = add_to_base(index, l);
    }
    else if (WN_operator_is(r, OPR_ARRAY))
    {
      *base  = add_to_base(base, WN_array_base(r));
      WN_array_base(r) = WN_Zerocon(WN_rtype(r));
      *index = add_to_base(index, addr);
    }
    else if (WN_operator_is(l, OPR_ARRAY))
    {
      *base  = add_to_base(base, WN_array_base(l));
      WN_array_base(l) = WN_Zerocon(WN_rtype(l));
      *index = add_to_base(index, addr);
    }
    else if ((WN_operator_is(r, OPR_ADD)  ||  WN_operator_is(r, OPR_SUB))
       && WN_operator_is(l, OPR_INTCONST))
    {
      lower_to_base_index(r, base, index);
      *index = add_to_base(index, l);
    }
    else if ((WN_operator_is(l, OPR_ADD)  ||  WN_operator_is(l, OPR_SUB)) 
       && WN_operator_is(r, OPR_INTCONST))
    {
      lower_to_base_index(l, base, index);
      *index = add_to_base(index, r);
    }
    else
    {
      // Give up
      *base  = add_to_base(base, addr);
      *index = add_to_base(index, WN_Zerocon(WN_rtype(addr)));
    }
    break;

  case OPR_SUB:
    l = WN_kid0(addr);
    r = WN_kid1(addr);

    if (baseAddress(l))
    {
      *base  = add_to_base(base, l);
      *index = sub_from_base(index, r);
    }
    else if (baseAddress(r))
    {
      *base  = sub_from_base(base, r);
      *index = add_to_base(index, l);
    }
    else if (WN_operator_is(r, OPR_ARRAY))
    {
      *base  =  sub_from_base(base, WN_array_base(r));
      WN_array_base(r) = WN_Zerocon(WN_rtype(r));
      *index =  add_to_base(index, addr);
    }
    else if (WN_operator_is(l, OPR_ARRAY))
    {
      *base  = add_to_base(base, WN_array_base(l));
      WN_array_base(l) = WN_Zerocon(WN_rtype(l));
      *index = add_to_base(index, addr);
    }
    else if ((WN_operator_is(r, OPR_ADD) ||  WN_operator_is(r, OPR_SUB))
       && WN_operator_is(l, OPR_INTCONST))
    {
      lower_to_base_index(r, base, index);
      *base  = WN_Neg(WN_rtype(*base),*base);
      *index = sub_from_base(&l,*index);
    }
    else if ((WN_operator_is(l, OPR_ADD) || WN_operator_is(l, OPR_SUB)) 
       && WN_operator_is(r, OPR_INTCONST))
    {
      lower_to_base_index(l, base, index);
      *index = sub_from_base(index, r);
    }
    else
    {
      // Give up
      *base  = add_to_base(base, addr);
      *index = add_to_base(index, WN_Zerocon(WN_rtype(addr)));
    }
    break;
  default:
    // Give up
    *base  = add_to_base(base, addr);
    *index = add_to_base(index, WN_Zerocon(WN_rtype(addr)));
    break;
  }
}


/*
 * Describe a leaf expression (see Make_Leaf/Load_Leaf).
 */
typedef enum {LEAF_IS_CONST, LEAF_IS_INTCONST, LEAF_IS_PREG} LEAF_KIND;
typedef struct {
  LEAF_KIND kind;
  TYPE_ID type;
  union {
    PREG_NUM n;
    INT64 intval;
    TCON tc;
  } u;
} LEAF;

/* ====================================================================
 *
 * LEAF Make_Leaf(WN *block, WN *tree, TYPE_ID type)
 *
 * Make an aribtrary expression tree into a leaf.
 * If the expression is an integer or floating point constant
 * no transformation is made. However, other expressions are stored
 * into a PREG.
 *
 * Make_Leaf is used in place of AssignExpr when performing a sort
 * of "poor man's" CSE, and you want to avoid creating unnecessary
 * PREGs for constants (which can also thwart the simplifier).
 *
 * ==================================================================== */

static LEAF Make_Leaf(WN *block, WN *tree, TYPE_ID type)
{
  LEAF leaf;
  leaf.type = type;
  switch (WN_operator(tree)) {
  case OPR_CONST:
    leaf.kind = LEAF_IS_CONST;
    leaf.u.tc = Const_Val(tree);
    WN_Delete(tree);
    break;
  case OPR_INTCONST:
    leaf.kind = LEAF_IS_INTCONST;
    leaf.u.intval = WN_const_val(tree);
    WN_Delete(tree);
    break;
  default:
    leaf.kind = LEAF_IS_PREG;
    leaf.u.n = AssignExpr(block, tree, type);
    break;
  }
  return leaf;
}

/* ====================================================================
 *
 * WN *Load_Leaf(const LEAF &leaf)
 *
 * Generate whirl to load the value of a leaf expression created by
 * Make_Leaf().
 *
 * ==================================================================== */

static WN *Load_Leaf(const LEAF &leaf)
{
  switch (leaf.kind) {
  case LEAF_IS_CONST:
    return Make_Const(leaf.u.tc);
  case LEAF_IS_INTCONST:
    return WN_CreateIntconst(OPR_INTCONST, leaf.type, MTYPE_V, leaf.u.intval);
  case LEAF_IS_PREG:
    return WN_LdidPreg(leaf.type, leaf.u.n);
  }
  FmtAssert(FALSE, ("unhandled leaf kind in Load_Leaf"));
  /*NOTREACHED*/
}

/* ====================================================================
 *
 * void lower_complex_expr(WN *block, WN *tree, LOWER_ACTIONS actions,
 *         WN **realpart, WN **imagpart)
 *
 * Split complex-type expression <tree> into its lowered real and
 * imaginary parts.  <actions> must include LOWER_COMPLEX.  Note that
 * like the other lowering functions, this one destroys <tree>.  Upon
 * return, *<realpart> points to the real part, and *<imagpart> points
 * to the imaginary part.
 *
 * ==================================================================== */

static void lower_complex_expr(WN *block, WN *tree, LOWER_ACTIONS actions,
             WN **realpart, WN **imagpart)
{
  TYPE_ID type;


  Is_True(OPCODE_is_expression(WN_opcode(tree)),
    ("expected expression node, not %s", OPCODE_name(WN_opcode(tree))));
  Is_True(Action(LOWER_COMPLEX), ("actions does not contain LOWER_COMPLEX"));
#ifndef TARG_X8664 // could be MTYPE_F8 that contains a MTYPE_C4
  Is_True((MTYPE_is_complex(WN_rtype(tree))),
    ("expected complex-type node, not %s",
     OPCODE_name(WN_opcode(tree))));
#endif
  type = Mtype_complex_to_real(WN_rtype(tree));

 /*
  *  Complex Arithmetic Notation
  *
  *  Let R(c) = real part of c
  *      I(c) = imaginary part of c
  *
  *  then, if z is complex it can be represented as follows
  *
  *      z= R(z) + I(z)i
  *
  */
  switch (WN_operator(tree))
  {
  case OPR_LDID:
    {
     /*
      *  we must create a new ST of type real
      *  we implicitly assume the storage of complex is
      *    z =  {  R(z) , I(z) }
      */
      TY_IDX    beTY   = MTYPE_To_TY(type);
      WN_OFFSET offset = WN_load_offset(tree);

      if (WN_class(tree) == CLASS_CONST && offset == 0)
      {
  TCON  val = WN_val(tree);
  TYPE_ID valType = WN_val_type(tree);

  if (WN_rtype(tree) == valType)
  {
    *realpart = Make_Const( Extract_Complex_Real(val));
    *imagpart = Make_Const( Extract_Complex_Imag(val));
    break;
  }
      }

      *realpart = WN_Ldid(type, offset,
        coerceST(WN_st(tree), type),
        beTY);

      *imagpart = WN_Ldid(type,
        coerceOFFSET(tree, type, offset),
        coerceST(WN_st(tree), type),
        beTY);
    }
    break;

  case OPR_ILOAD:
    {
     /*
      *  we implicitly assume the storage of complex is
      *    z =  {  R(z) , I(z) }
      *
      *  The LOWER_BASE_INDEX will try to split the address into a
      *  base and index. The index is put in a preg (and reused) while
      *  the base is cloned.
      */
      WN_OFFSET offset = WN_load_offset(tree);

      if (Action(LOWER_BASE_INDEX))
      {
  WN  *addr, *base, *index;
  LEAF  indexN;

  base = index= NULL;
  lower_to_base_index(WN_kid0(tree), &base, &index) ;

  base = lower_expr(block, base, actions);
  index = lower_expr(block, index, actions);

  indexN = Make_Leaf(block, index, Pointer_type);
  
  addr = WN_Add(Pointer_type,
          Load_Leaf(indexN),
          lower_copy_tree(base, actions)); 

  *realpart = WN_Iload(type,
           offset,
           coerceTY(WN_ty(tree), type),
           addr);

  addr = WN_Add(Pointer_type, Load_Leaf(indexN), base);

  *imagpart = WN_Iload(type,
           coerceOFFSET(tree, type, offset),
           coerceTY(WN_ty(tree), type),
           addr);
      }
      else
      {
  WN  *addr;
  LEAF  addrN;
  
  addr = lower_expr(block, WN_kid0(tree), actions);
  
  addrN = Make_Leaf(block, addr, Pointer_type);
  
  *realpart = WN_Iload(type,
           offset,
           coerceTY(WN_ty(tree), type),
           Load_Leaf(addrN));
  
  *imagpart = WN_Iload(type,
           coerceOFFSET(tree, type, offset),
           coerceTY(WN_ty(tree), type),
           Load_Leaf(addrN));
      }
      lower_complex_maps(tree, *realpart, *imagpart, actions);
    }
    break;

  case OPR_ILOADX:
    Is_True(FALSE, ("unexpected complex ILOADX"));
    break;

  case OPR_NEG:
    {
     /*
      *    -z  = -R(z) + -I(z)i
      */
      WN  *rz, *iz;

      lower_complex_expr(block, WN_kid0(tree), actions, &rz, &iz);
      *realpart = WN_Neg( type, rz);
      *imagpart = WN_Neg( type, iz);

    }
    break;

  case OPR_ADD:
    {
      WN  *rz, *rw, *iz, *iw;
     /*
      *    z + w = (R(z) + R(w)) + (I(z) + I(w))i
      */

      lower_complex_expr(block, WN_kid0(tree), actions, &rz, &iz);
      lower_complex_expr(block, WN_kid1(tree), actions, &rw, &iw);
 
      *realpart = WN_Add( type, rz, rw);
      *imagpart = WN_Add( type, iz, iw);
    }
    break;

  case OPR_SUB:
    {
     /*
      *    z - w = (R(z) - R(w)) + (I(z) - I(w))i
      */
      WN  *rz, *rw, *iz, *iw;

      lower_complex_expr(block, WN_kid0(tree), actions, &rz, &iz);
      lower_complex_expr(block, WN_kid1(tree), actions, &rw, &iw);
 
      *realpart = WN_Sub( type, rz, rw);
      *imagpart = WN_Sub( type, iz, iw);
    }
    break;

  case OPR_MPY:
    {
     /*
      *    z * w = (R(z)*R(w) - I(z)*I(w)) + (R(z)*I(w)+ R(w)*I(z))i
      *
      */
      WN  *rz, *rw, *iz, *iw;
      LEAF  rzN, rwN, izN, iwN;

      lower_complex_expr(block, WN_kid0(tree), actions, &rz, &iz);
      lower_complex_expr(block, WN_kid1(tree), actions, &rw, &iw);

      rzN = Make_Leaf(block, rz, type);
      rwN = Make_Leaf(block, rw, type);
      izN = Make_Leaf(block, iz, type);
      iwN = Make_Leaf(block, iw, type);

      *realpart = WN_Sub(type,
       WN_Mpy(type, Load_Leaf(rzN), Load_Leaf(rwN)),
       WN_Mpy(type, Load_Leaf(izN), Load_Leaf(iwN)));

      *imagpart = WN_Add(type,
       WN_Mpy(type, Load_Leaf(rzN), Load_Leaf(iwN)),
       WN_Mpy(type, Load_Leaf(rwN), Load_Leaf(izN)));
    }
    break;

  case OPR_COMPLEX:
    /*
     *  Create a complex number from two real children, ie.
     *    z = CMPLX(x,y)
     *    z = (x) + (y) i
     */
    *realpart = lower_expr(block, WN_kid0(tree), actions);
    *imagpart = lower_expr(block, WN_kid1(tree), actions);
    // Fix for bug# 414
    //WN_Delete(tree);
    break;

  case OPR_MADD:
  case OPR_MSUB:
  case OPR_NMADD:
  case OPR_NMSUB:
    Is_True( FALSE, ("unexpected complex madd"));
    break;

  case OPR_CVT:
    {
      WN  *rz, *iz;
      TYPE_ID desc = WN_desc(tree);

      lower_complex_expr(block, WN_kid0(tree), actions, &rz, &iz);
 
      *realpart = WN_Cvt( type, Mtype_complex_to_real(desc), rz);
      *imagpart = WN_Cvt( type, Mtype_complex_to_real(desc), iz);
    }
    break;

  case OPR_CONST:
    {
     /*
      * extract the real and imaginary parts of the complex number
      */
      TCON  val = Const_Val(tree);

      *realpart = Make_Const( Extract_Complex_Real(val));
      *imagpart = Make_Const( Extract_Complex_Imag(val));

      // Fix for Bug# 496
      //WN_Delete(tree);
    }
    break;

  case OPR_RSQRT:
    {
      TYPE_ID desc = WN_desc(tree);
      WN  *div;

      div = WN_Div(desc,
       WN_Floatconst(desc, 1.0),
       WN_Sqrt(desc, WN_kid0(tree)));

      lower_complex_expr(block, div, actions, realpart, imagpart);

      WN_Delete(tree);
    }
    break;

  case OPR_SQRT:
    lower_complex_emulation(block, tree, actions, realpart, imagpart);
    break;


  case OPR_PAREN:
    {
      lower_complex_expr(block, WN_kid0(tree), actions, realpart, imagpart);

      *realpart = WN_Paren(type, *realpart);
      *imagpart = WN_Paren(type, *imagpart);
      WN_Delete(tree);
    }
    break;

  case OPR_ARRAY:
    break;

  case OPR_RECIP:
    {
      /* TODO_FEEDBACK: Get frequencies right, especially when we
       * create an IF statement. We have to assume half the frequency
       * goes to the then part and half goes to the else part; there's
       * no other information available, is there?
       */

      LEAF  rzN, izN;
      {
       /*
  *  assign pregs to their corresponding expressions
  *  Since the expressions will be reused, this avoids building a DAG
  */
  WN  *rz, *iz; 
    
  lower_complex_expr(block, WN_kid0(tree), actions, &rz, &iz);
      
  rzN = Make_Leaf(block, rz, type);
  izN = Make_Leaf(block, iz, type);
      }
    
      if (Fast_Complex_Allowed)
      {
       /*
  *   1 / z
  *    real = (   R(z) ) /  ( R(z)**2 + I(z)**2 )
  *    imag = ( - I(z) ) /  ( R(z)**2 + I(z)**2 )
  *
  */
  LEAF  denomN;
  WN  *rz2, *iz2, *add;
    
  rz2 = WN_Mpy(type, Load_Leaf(rzN), Load_Leaf(rzN));
  iz2 = WN_Mpy(type, Load_Leaf(izN), Load_Leaf(izN));
  add = WN_Add(type, rz2, iz2);
  denomN = Make_Leaf(block, add, type);
    
  *realpart = WN_Div(type, Load_Leaf(rzN), Load_Leaf(denomN));
  *imagpart = WN_Neg(type,
         WN_Div(type, Load_Leaf(izN), Load_Leaf(denomN)));
      }
      else
      {
       /*
  *   1 / z
  *    real = (   R(z) ) /  ( R(z)**2 + I(z)**2 )
  *    imag = ( - I(z) ) /  ( R(z)**2 + I(z)**2 )
  *
  *    After factoring out max( |R(z)| , |I(z)| )
  *
  * | R(z) | >  | I(z) |
  *
  * Let x = I(z) / R(z)
  * real = ( 1 )    /  ( R(z) + I(z)*x)
  * imag = ( - x )  /  ( R(z) + I(z)*x)
  *
  * | I(z) | >  | R(z) |
  *
  * Let x = R(z) / I(z)
  * real = ( x )   /  ( R(z)*x + I(z) )
  * imag = (-1 )   /  ( R(z)*x + I(z) )
  *
  */
  WN    *if_then, *if_else, *IF;
  PREG_NUM  realpartN, imagpartN;
  LEAF    xN, arzN, aizN;
    
  {
    WN  *numer, *denom, *div;
    
    arzN = Make_Leaf(block,
         WN_Abs(type, Load_Leaf(rzN)),
             type);
    
    aizN = Make_Leaf(block,
             WN_Abs(type, Load_Leaf(izN)),
             type);
    
   /*
    *  numer =  | R(w) | >  | I(w) |  ?  I(w) : R(x)
    *  denom =  | R(w) | >  | I(w) |  ?  R(w) : I(x)
    *
    *  Let x = numer / denom
    */
    numer = WN_Select(type,
          WN_GT(type, Load_Leaf(arzN), Load_Leaf(aizN)),
              Load_Leaf(izN),
              Load_Leaf(rzN));
    denom = WN_Select(type,
          WN_GT(type, Load_Leaf(arzN), Load_Leaf(aizN)),
              Load_Leaf(rzN),
                Load_Leaf(izN));

    div = WN_Div(type, numer, denom);

    xN = Make_Leaf(block, div, type);
  }

  if_then = WN_CreateBlock();
  {
   /*
    * scale = ( R(z) + I(z)*x )
    * real = ( 1 )    /  scale
    * imag = ( - x )  /  scale
    */
    WN  *scale, *div;
    LEAF  scaleN;
    
    scale = WN_Add(type,
       Load_Leaf(rzN),
       WN_Mpy(type, Load_Leaf(izN), Load_Leaf(xN)));
    scaleN = Make_Leaf(if_then, scale, type);

    div = WN_Inverse(type, Load_Leaf(scaleN));

    realpartN = AssignExpr(if_then, div, type);

    div =  WN_Div(type,
      WN_Neg(type, Load_Leaf(xN)),
      Load_Leaf(scaleN));
    
    imagpartN = AssignExpr(if_then, div, type);
  }

  if_else = WN_CreateBlock();
  {
   /*
    * scale = ( R(z)*x + I(z) )
    * real = ( x )   /  scale
    * imag = ( 1 )   /  scale
    */
    WN  *scale, *div, *stid;
    LEAF  scaleN;
    ST  *preg = MTYPE_To_PREG(type);
    
    scale = WN_Add(type,
       WN_Mpy(type, Load_Leaf(rzN), Load_Leaf(xN)),
       Load_Leaf(izN));
    scaleN = Make_Leaf(if_else, scale, type);

    div =  WN_Div(type, Load_Leaf(xN), Load_Leaf(scaleN));
    stid = WN_StidIntoPreg(type, realpartN, preg, div);
    WN_Set_Linenum(stid, current_srcpos);  // Bug 1268
    WN_INSERT_BlockLast(if_else, stid);
    
    div = WN_Neg(type,WN_Inverse(type, Load_Leaf(scaleN)));
    stid = WN_StidIntoPreg(type, imagpartN, preg, div);
    WN_Set_Linenum(stid, current_srcpos);  // Bug 1268
    WN_INSERT_BlockLast(if_else, stid);
  }
    
  IF =  WN_CreateIf(WN_GT(type, Load_Leaf(arzN), Load_Leaf(aizN)),
        if_then, if_else);

  if ( Cur_PU_Feedback ) {
    Cur_PU_Feedback->Annot( IF, FB_EDGE_BRANCH_TAKEN, FB_FREQ_UNKNOWN );
    Cur_PU_Feedback->Annot( IF, FB_EDGE_BRANCH_NOT_TAKEN,
          FB_FREQ_UNKNOWN );
  }

  WN_INSERT_BlockLast(block, lower_if(block, IF, actions));
    
  *realpart = WN_LdidPreg(type, realpartN);
  *imagpart = WN_LdidPreg(type, imagpartN);
      }
      WN_Delete(tree);
    }
    break;

  case OPR_DIV:
    {
     /*
      *  assign pregs to their corresponding expressions
      *  Since the expressions will be reused, this avoids building a DAG
      */
      WN  *rz, *rw, *iz, *iw; 
      LEAF  rzN, rwN, izN, iwN;

      lower_complex_expr(block, WN_kid0(tree), actions, &rz, &iz);
      lower_complex_expr(block, WN_kid1(tree), actions, &rw, &iw);
  
      rzN = Make_Leaf(block, rz, type);
      izN = Make_Leaf(block, iz, type);
      rwN = Make_Leaf(block, rw, type);
      iwN = Make_Leaf(block, iw, type);

      if (Fast_Complex_Allowed)
      {
       /*
  *   z / w
  *    real = (R(z)*R(w) + I(z)*I(w) /  ( R(w)**2 + I(w)**2 )
  *    imag = (I(z)*R(w) - R(z)*I(w) /  ( R(w)**2 + I(w)**2 )
  *
  */
  LEAF  denomN;
  {
    WN  *rw2, *iw2, *add;
    
    rw2 = WN_Mpy(type,Load_Leaf(rwN), Load_Leaf(rwN));
    iw2 = WN_Mpy(type, Load_Leaf(iwN), Load_Leaf(iwN));
    add = WN_Add(type, rw2, iw2);
    denomN = Make_Leaf(block, add, type);
  }
  {
    WN  *rzrw, *iziw;
    
    rzrw = WN_Mpy(type, Load_Leaf(rzN), Load_Leaf(rwN));
    iziw = WN_Mpy(type, Load_Leaf(izN), Load_Leaf(iwN));
    *realpart = WN_Div(type,
           WN_Add(type, rzrw, iziw),
           Load_Leaf(denomN));
  }
  {
    WN  *rziw, *izrw;
    
    izrw = WN_Mpy(type, Load_Leaf(izN), Load_Leaf(rwN));
    rziw = WN_Mpy(type, Load_Leaf(rzN), Load_Leaf(iwN));
    *imagpart = WN_Div(type,
           WN_Sub(type, izrw, rziw),
           Load_Leaf(denomN));
  }
      }
      else
      {
       /*
  *   z / w
  *    real = (R(z)*R(w) + I(z)*I(w) /  ( R(w)**2 + I(w)**2 )
  *    imag = (I(z)*R(w) - R(z)*I(w) /  ( R(w)**2 + I(w)**2 )
  *
  *    After factoring out max( |R(w)| , |I(w)| )
  *
  * | R(w) | >  | I(w) |
  *
  * Let x = I(w) / R(w)
  *   real = ( R(z) + I(z)*x ) /  ( R(w) + I(w)*x )
  *   imag = ( I(z) - R(z)*x ) /  ( R(w) + I(w)*x )
  *
  * | I(w) | >  | R(w) |
  *
  * Let x = R(w) / I(w)
  *   real = ( R(z)*x + I(z) ) /  ( R(w)*x + I(w) )
  *   imag = ( I(z)*x - R(z) ) /  ( R(w)*x + I(w) )
  *
  */
  WN    *if_then, *if_else, *IF;
  LEAF    xN, arwN, aiwN;
  PREG_NUM  realpartN, imagpartN;
  {
    WN  *numer, *denom, *div;
    
    arwN = Make_Leaf(block,
             WN_Abs(type, Load_Leaf(rwN)),
             type);
    aiwN = Make_Leaf(block,
             WN_Abs(type, Load_Leaf(iwN)),
             type);
    
   /*
    *  numer =  | R(w) | >  | I(w) |  ?  I(w) : R(x)
    *  denom =  | R(w) | >  | I(w) |  ?  R(w) : I(x)
    *
    *  Let x = numer / denom
    */
    numer = WN_Select(type,
          WN_GT(type, Load_Leaf(arwN), Load_Leaf(aiwN)),
              Load_Leaf(iwN),
          Load_Leaf(rwN));
    denom = WN_Select(type,
          WN_GT(type, Load_Leaf(arwN), Load_Leaf(aiwN)),
              Load_Leaf(rwN),
              Load_Leaf(iwN));
    div = WN_Div(type, numer, denom);
    xN = Make_Leaf(block, div, type);
  }

  if_then = WN_CreateBlock();
  {
    WN    *scale;
    LEAF  scaleN;
    
    scale = WN_Add(type,
       Load_Leaf(rwN),
       WN_Mpy(type, Load_Leaf(iwN), Load_Leaf(xN)));
    scaleN = Make_Leaf(if_then, scale, type);
    {
     /*
          *  real = ( R(z)   + I(z)*x )  / scale
          */
          WN  *numer, *div;
    
          numer = WN_Add(type,
           Load_Leaf(rzN),
           WN_Mpy(type, Load_Leaf(izN), Load_Leaf(xN)));
      div = WN_Div(type, numer, Load_Leaf(scaleN));
      realpartN = AssignExpr(if_then, div, type);
    }
    {
     /*
      *  imag = ( I(z) - R(z)*x ) /  scale
          */
          WN  *numer, *div;
    
      numer = WN_Sub(type,
             Load_Leaf(izN),
         WN_Mpy(type, Load_Leaf(rzN), Load_Leaf(xN)));
      div = WN_Div(type, numer, Load_Leaf(scaleN));
          imagpartN = AssignExpr(if_then, div, type);
    }
  }
    
  if_else = WN_CreateBlock();
  {
    WN  *scale;
    LEAF  scaleN;
    ST  *preg = MTYPE_To_PREG(type);

    scale = WN_Add(type,
       WN_Mpy(type, Load_Leaf(rwN), Load_Leaf(xN)),
           Load_Leaf(iwN));
    scaleN = Make_Leaf(if_else, scale, type);
    {
     /*
          *  real = ( R(z)*x + I(z) ) /  scale
          *  store away result in an already defined preg
          */
          WN  *numer, *div, *stid;
    
          numer =  WN_Add(type,
              WN_Mpy(type, Load_Leaf(rzN), Load_Leaf(xN)),
              Load_Leaf(izN));
      div = WN_Div(type, numer, Load_Leaf(scaleN));
      stid = WN_StidIntoPreg(type, realpartN, preg, div);
      WN_Set_Linenum(stid, current_srcpos);  // Bug 1268
      WN_INSERT_BlockLast(if_else, stid);
    }
    {
     /*
      *  imag = ( I(z)*x - R(z) ) /  scale
          *  store away result in an already defined preg
          */
          WN  *numer, *div, *stid;
    
          numer = WN_Sub(type,
             WN_Mpy(type, Load_Leaf(izN), Load_Leaf(xN)),
             Load_Leaf(rzN));
      div = WN_Div(type, numer, Load_Leaf(scaleN));
      stid = WN_StidIntoPreg(type, imagpartN, preg, div);
      WN_Set_Linenum(stid, current_srcpos);  // Bug 1268
      WN_INSERT_BlockLast(if_else, stid);
    }
  }
    
  IF =  WN_CreateIf(WN_GT(type, Load_Leaf(arwN), Load_Leaf(aiwN)),
        if_then, if_else);

  if ( Cur_PU_Feedback ) {
    Cur_PU_Feedback->Annot( IF, FB_EDGE_BRANCH_TAKEN, FB_FREQ_UNKNOWN );
    Cur_PU_Feedback->Annot( IF, FB_EDGE_BRANCH_NOT_TAKEN,
          FB_FREQ_UNKNOWN );
  }
    
  WN_INSERT_BlockLast(block, lower_if(block, IF, actions));
    
  *realpart = WN_LdidPreg(type, realpartN);
  *imagpart = WN_LdidPreg(type, imagpartN);
      }
      WN_Delete(tree);
    }
    break;

  case OPR_RND:
  case OPR_TRUNC:
  case OPR_MOD:
  case OPR_REM:
  case OPR_ABS:
    Fail_FmtAssertion("unexpected complex op (%s)",
          OPCODE_name(WN_opcode(tree)));
    /*NOTREACHED*/

  case OPR_INTRINSIC_OP:
    {
      INTRINSIC     id = (INTRINSIC) WN_intrinsic(tree);

      switch(id)
      {
      case INTRN_C4CONJG:
      case INTRN_C8CONJG:
      case INTRN_CQCONJG:
  {
    WN  *iz;

    lower_complex_expr(block, WN_actual(tree, 0), actions,
           realpart, &iz);

    *imagpart = WN_Neg(type, iz);

    WN_Delete(tree);
  }
        break;

  //*****************************************************************
  //
  // N.B. Any complex intrinsic which does not have an emulation must
  // appear in the list below.
  //

  //       case INTRN_F4CIS:
  //case INTRN_F8CIS:
  //case INTRN_FQCIS:
  //  actions |= LOWER_INTRINSIC;
    /* Fall Through */

      default:
  if (INTRN_is_actual(WN_intrinsic(tree)))
  {
    tree = lower_intrinsic(block, tree, actions);
  }
  else
  {
    lower_complex_emulation(block, tree, actions, realpart, imagpart);
  }
        break;
      }
    }
    break;

  case OPR_SELECT:
    {
       WN *r1, *i1;
       WN *r2, *i2;
       LEAF cond;
       
       cond = Make_Leaf(block, WN_kid0(tree), WN_rtype(WN_kid0(tree)));
       
       lower_complex_expr(block, WN_kid1(tree), actions, &r1, &i1);
       lower_complex_expr(block, WN_kid2(tree), actions, &r2, &i2);
       
       *realpart = WN_Select(type, Load_Leaf(cond), r1, r2);
       *imagpart = WN_Select(type, Load_Leaf(cond), i1, i2);
    }
    break;

  case OPR_PARM:
    lower_complex_expr(block, WN_kid0(tree), actions, realpart, imagpart);
    break;
  }
}




/* ====================================================================
 *
 * void lower_quad_expr(WN *block, WN *tree, LOWER_ACTIONS actions,
 *         WN **hipart, WN **lopart)
 *
 * ==================================================================== */

static void lower_quad_expr(WN *block, WN *tree, LOWER_ACTIONS actions,
             WN **hipart, WN **lopart)
{
  TYPE_ID type =  MTYPE_F8;


  Is_True(OPCODE_is_expression(WN_opcode(tree)),
    ("expected expression node, not %s", OPCODE_name(WN_opcode(tree))));
  Is_True(actions & LOWER_QUAD, ("actions does not contain LOWER_QUAD"));
  Is_True((MTYPE_is_quad(WN_rtype(tree))),
    ("expected quad-type node, not %s", OPCODE_name(WN_opcode(tree))));
 /*
  *  first lower the quad as expressions.
  *
  *  This will create and serialize a call chain
  *  that should always return a leaf that can be decomposed
  */
  tree = lower_expr(block, tree, actions);

  switch (WN_operator(tree))
  {
  case OPR_LDID:
    {
     /*
      *  we must create a new ST of type real*8
      */
      TY_IDX    beTY   = MTYPE_To_TY(type);
      WN_OFFSET offset = WN_load_offset(tree);

      *hipart = WN_Ldid(type, offset,
      coerceST(WN_st(tree), type), beTY);

      *lopart = WN_Ldid(type,
      coerceOFFSET(tree, type, offset),
      coerceST(WN_st(tree), type),
      beTY);

      lower_quad_maps(tree, *hipart, *lopart, actions);
    }
    break;

  case OPR_ILOAD:
    {
      WN  *addr;
      WN_OFFSET offset = WN_load_offset(tree);
      LEAF  addrN;

      addr = lower_expr(block, WN_kid0(tree), actions);

      addrN = Make_Leaf(block, addr, Pointer_type);

      *hipart = WN_Iload(type,
       offset,
       coerceTY(WN_ty(tree), type),
       Load_Leaf(addrN));

      *lopart = WN_Iload(type,
       coerceOFFSET(tree, type, offset),
       coerceTY(WN_ty(tree), type),
       Load_Leaf(addrN));

      lower_quad_maps(tree, *hipart, *lopart, actions);
    }
    break;

  case OPR_CONST:
    {
      TCON  val = Const_Val(tree);

      *hipart = Make_Const( Extract_Quad_Hi(val));
      *lopart = Make_Const( Extract_Quad_Lo(val));

    }
    break;

  case OPR_PAREN:
    {
     /*
      * preserve the parens
      */
      lower_quad_expr(block, WN_kid0(tree), actions, hipart, lopart);

      *hipart = WN_Paren(type, *hipart);
      *lopart = WN_Paren(type, *lopart);
    }
    break;

  case OPR_PARM:
    lower_quad_expr(block, WN_kid0(tree), actions, hipart, lopart);
    break;

  case OPR_ILOADX:
    Is_True( FALSE, ("unexpected QUAD ILOADX"));
    break;

  default:
    Is_True((FALSE),
      ("lower_quad_expr didn't %s", OPCODE_name(WN_opcode(tree))));
  }
}




static WN *WN_Coerce(TYPE_ID dst, WN *expr)
{
  TYPE_ID src= WN_rtype(expr);

  if (MTYPE_size_min(dst) == MTYPE_size_min(src))
    return expr;
#ifdef KEY // avoid ambiguity
  if (src == MTYPE_I4 && dst == MTYPE_U8)
    dst = MTYPE_I8;
  if (src == MTYPE_U4 && dst == MTYPE_I8)
    src = MTYPE_I4;
#endif
  return WN_Cvt(src, dst, expr);
}




/* ====================================================================
 *
 * WN *lower_linearize_array_addr(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on ARRAY expr <tree>,
 * returning linearized address expression tree.  <actions> must
 * contain LOWER_ARRAY.
 *
 *  We are assuming that the index and array_dim information are unique
 *
 * Note: If the element size is < 0, it means that we have a non-contiguous F90
 *       array. These are linearized slightly differently. Instead of
 *       the extent children being extents, they are stride multiplier factors. 
 *       The sum of the products of the index and stride multiplier is scaled
 *       by -element_size to get the offset from the base.  
 * ==================================================================== */

static WN *lower_linearize_array_addr(WN *block, WN *tree,
              LOWER_ACTIONS actions)
{
  TYPE_ID rtype = WN_rtype(tree);
  WN    *result, *product;
  INT32         i;
  BOOL          is_non_contig=FALSE;
  WN_ESIZE      element_size;

  Is_True((WN_operator_is(tree,OPR_ARRAY)),
    ("expected ARRAY node, not %s", OPCODE_name(WN_opcode(tree))));
  Is_True(Action(LOWER_ARRAY), ("actions doesn't contain LOWER_ARRAY"));
  Is_True(WN_num_dim(tree) > 0, ("WN_num_dim of ARRAY node not > 0"));

  const INT32 n = WN_num_dim(tree);
  element_size = WN_element_size(tree);
  if (element_size < 0) {
     is_non_contig = TRUE;
     element_size = -element_size;
  }
  if (element_size == 0) {
    // This is an array of empty struct
    element_size = 1;
  }
  
#ifdef TARG_X8664
  BOOL do_reassociate = FALSE;
#endif

  if (is_non_contig)
  {
     WN *stride_mult;
     
     result = WN_Coerce(rtype, WN_array_index(tree, n-1));
     stride_mult = WN_Coerce(rtype, WN_array_dim(tree, n-1));
     result = WN_Mpy(rtype,result,stride_mult);

     for (i = n-2; i >= 0; i--) {
  product = WN_Coerce(rtype, WN_array_index(tree, i));
  stride_mult = WN_Coerce(rtype, WN_array_dim(tree, i));
  product = WN_Mpy(rtype,product,stride_mult);
  result = WN_Add(rtype,result,product);
     }
  }
  else
  {
#ifdef TARG_X8664
    result = NULL;
    do_reassociate = FALSE;

    if( n > 1 && loop_info_stack[current_loop_nest_depth] != NULL ){
      WN* idv = WN_array_index( tree, n-1 );
      while( idv != NULL ){
  if( WN_has_sym(idv) )
    break;
  if( WN_kid_count(idv) == 0 ){
    idv = NULL;
    break;
  }
  idv = WN_kid0( idv );
      }

      if( idv != NULL ){
  do_reassociate = 
    WN_st_idx( idv ) == WN_st_idx( loop_info_stack[current_loop_nest_depth] );
  // for stride cases ...
  do_reassociate |= ( WN_class(idv) == CLASS_PREG );

  // You can turn me off if you don't like me :(
  //do_reassociate = FALSE;
      }
    }

    if( !do_reassociate ){
      result = WN_Coerce(rtype, WN_array_index(tree, n-1));
    }
#else
   /*
    *  result <- index[n-1]
    */
    result = WN_Coerce(rtype, WN_array_index(tree, n-1));
#endif
     
   /*
    *  result <- result + index[i] * ( dim[n-1] * dim[n-2] ... dim[i+1] )
    */
    for (i = n-2; i >= 0; i--)
    {
      INT32 m;
      WN  *mpy;
       
      product = WN_Coerce(rtype, lower_copy_tree(WN_array_dim(tree, n-1),
             actions));
      for (m=n-2; m>i; m--)
      {
  product = WN_Mpy(rtype,
       product,
       WN_Coerce(rtype, lower_copy_tree(WN_array_dim(tree, m),
                actions)));
      }

      mpy = WN_Mpy(rtype,
       WN_Coerce(rtype, WN_array_index(tree,i)),
       product);
#ifdef TARG_X8664
      if( result == NULL ){
  result = mpy;
  continue;
      }
#endif
      result = WN_Add(rtype, result, mpy);
    }
  }  
  

  /*
   *  result <- base + result * elm_size
   */
  {
    WN  *elm_size;

    elm_size = WN_Intconst(rtype, element_size);
    result = WN_Add(rtype,
        WN_array_base(tree),
        WN_Mpy(rtype, result, elm_size));
#ifdef TARG_X8664
    if( do_reassociate ){
      /*
       *  result <- result +  index[n-1] * elm_size
       */
      WN* elm_size1 = WN_Intconst(rtype, element_size);
      WN* inv_wn = WN_Coerce(rtype, WN_array_index( tree, n-1 ));    
      result = WN_Add(rtype,
          result,
          WN_Mpy(rtype, inv_wn, elm_size1) );
    }
#endif
  }

  result = lower_expr(block, result, actions);

  WN_Delete(tree);      /* ARRAY node not used */

  return result;
}




/* ====================================================================
 *
 * WN *lower_unsigned_to_float(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on conversions <tree>,
 * returning lowered expression tree.
 *
 * ==================================================================== */

static WN *lower_unsigned_to_float(WN *block, WN *expr, TYPE_ID src,
           TYPE_ID dst, LOWER_ACTIONS actions)
{
  LEAF  srcNo, dstNo;


  /* Special case of U4. Don't do this special case if the cvt.d.l 
   * instruction is slow. This is true on the R5000 which emulates
   * this in software due to a chip bug.
   */
  if (src == MTYPE_U4 && !Slow_CVTDL)
  {
     BOOL  simp=  WN_Simplifier_Enable(FALSE);
     WN * r;

     r = WN_Cvt(MTYPE_I8,dst,WN_Cvt(MTYPE_U4, MTYPE_U8, expr));

     WN_Simplifier_Enable(simp);
     return (r);
  }


  /*
   *  store the expr into a preg to avoid building a dag
   *  src = expr;
   */
  srcNo = Make_Leaf(block, expr, src);

  /*
   *  dst = (signed cvt) src ;
   */
  {
    WN  *ldid, *cvt;

    ldid = Load_Leaf(srcNo);

    cvt  = WN_Cvt(MTYPE_complement(src), dst, ldid);

    dstNo = Make_Leaf(block, cvt, dst);
  }

  /*
   *  build the select tree that looks like
   *  (src < 0) ? dst + 2**N : dst
   *   where N is the src size
   */
  {
    WN  *rel, *add, *select;

    rel = WN_LT(MTYPE_complement(src),
    Load_Leaf(srcNo),
    WN_Zerocon(MTYPE_complement(src)));

    add = WN_Add(dst,
     Load_Leaf(dstNo),
     WN_ConstPowerOf2( dst, MTYPE_size_reg(src)));

    select = WN_Select(dst, rel, add, Load_Leaf(dstNo));
    return select;
  }
}




/* ====================================================================
 *
 * WN *lower_float_to_unsigned(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on conversions <tree>,
 * returning lowered expression tree.
 *
 * ==================================================================== */

static WN *lower_float_to_unsigned(WN *block, WN *expr, TYPE_ID src,
           TYPE_ID dst, LOWER_ACTIONS actions)
{
  LEAF srcNo;
  WN *trunc1,*r;
  WN *rel, *sub, *trunc2,*add;


  /* Two cases, dest type = U8, and dest type = U4 */
  if (dst == MTYPE_U4) {
     r = WN_Cvt(MTYPE_I8,dst,WN_Trunc(src,MTYPE_I8,expr));
  } else if (src==MTYPE_FQ) {  /* Need to do this this way because there
        * is no quad floor */
     
     /*
      *  store the expr into a preg to avoid building a dag
      * src = expr;
      */
     srcNo = Make_Leaf(block, expr, src);
     
     /*
      *  build the select tree that looks like
      *
      * (2**(N-1) <= src) ? : 2**(N-1) + TRUNC(src-2**(N-1)) : TRUNC(src)
      *
      * where N is the unsigned dst size
      */


     trunc1 = WN_Trunc(src, MTYPE_complement(dst), Load_Leaf(srcNo));
     
     sub = WN_Sub(src, Load_Leaf(srcNo),
      WN_ConstPowerOf2(src, MTYPE_size_reg(dst)-1));

     trunc2 =  WN_Trunc(src, MTYPE_complement(dst),sub);

     add = WN_Add(dst,trunc2,WN_ConstPowerOf2(dst, MTYPE_size_reg(dst)-1));
     
     rel = WN_LE(src,
     WN_ConstPowerOf2(src, MTYPE_size_reg(dst)-1),
     Load_Leaf(srcNo));
     
     r = WN_Cselect(dst,rel,add,trunc1);
  } else {
     
     /*
      *  store the expr into a preg to avoid building a dag
      * src = expr;
      */
     srcNo = Make_Leaf(block, expr, src);
     
     /*
      *  build the select tree that looks like
      *
      * (2**(N-1) <= src) ? : FLOOR(src-2**(N)) : TRUNC(src)
      *
      * where N is the unsigned dst size
      */


     trunc1 = WN_Trunc(src, MTYPE_complement(dst), Load_Leaf(srcNo));
     
     sub = WN_Sub(src, Load_Leaf(srcNo),
      WN_ConstPowerOf2(src, MTYPE_size_reg(dst)));

     trunc2 =  WN_Floor(src, MTYPE_complement(dst),sub);

     rel = WN_LE(src,
     WN_ConstPowerOf2(src, MTYPE_size_reg(dst)-1),
     Load_Leaf(srcNo));
     
     r = WN_Cselect(dst,rel,trunc2,trunc1);
  }
  return (r);
}




/* ====================================================================
 *
 * WN *lower_cvt(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on conversions <tree>,
 * returning lowered expression tree.
 *
 * ==================================================================== */

static WN *lower_cvt(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  WN  *expr, *cvt;
  TYPE_ID dst = WN_rtype(tree);
  TYPE_ID src = WN_desc(tree);

  expr = lower_expr(block, WN_kid0(tree), actions);
  WN_kid0(tree) = expr;

  if (   Targ_Lower_Unsigned_To_Float 
      && MTYPE_is_unsigned(src) && MTYPE_is_float(dst))
  {
    WN_Delete(tree);

    cvt = lower_unsigned_to_float(block, expr, src, dst, actions);

    return lower_expr(block, cvt, actions);
  }
  else if (   Targ_Lower_Float_To_Unsigned 
     && MTYPE_is_float(src) && MTYPE_is_unsigned(dst))
  {
    WN_Delete(tree);

    cvt = lower_float_to_unsigned(block, expr, src, dst, actions);

    return lower_expr(block, cvt, actions);
  }
  else if (   OPERATOR_is_compare(WN_operator(expr))
     && WN_rtype(expr) == MTYPE_B
     && (MTYPE_is_integral(dst) && src == MTYPE_B))
  {

    /* Optimize converts of MTYPE_B compares to integral values.
     */
    WN_Delete(tree);

    WN_set_rtype(expr, dst);

    return lower_expr(block, expr, actions);
  }
  
  return tree;
}



/* ====================================================================
 *
 * WN *lower_nary_madd(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 *  Create nary expression reassociate to produce MADDS
 *
 * ==================================================================== */

static WN *lower_nary_madd(WN *block, WN *tree, LOWER_ACTIONS actions)
{

  if (Enable_NaryExpr && Roundoff_Level >= ROUNDOFF_ASSOC)
  {
    TYPE_ID  type = WN_rtype(tree);

    tree = WN_ExprToNaryType(tree, type, 0);

    if (WN_nary_add(tree))
    {
      INT32 i;
      BOOL  found= TRUE;

     /*
      *  Find mpy then a non mpy under nary tree.
      *  The tree may change (ex. to a binary tree)
      */
      while(found)
      {
  INT32 mpyI, exprI, nmpyI, narympyI;
  WN  *mpy, *nmpy, *expr, *narympy;
        INT32 num_kids = WN_kid_count(tree);

  found=  FALSE;
  mpy  =  nmpy  = expr = narympy  = NULL;
  mpyI =  exprI = nmpyI = narympyI = 0;

  for(i = 0; i < num_kids; i++)
  {
    WN  *actual= WN_actual(tree, i);

    if (WN_operator_is(actual, OPR_MPY))
    {
      mpyI = i;
      mpy = actual;
    }
    else if (WN_operator_is(actual, OPR_NEG)  &&
       WN_operator_is(WN_kid0(actual), OPR_MPY))
    {
      nmpyI = i;
      nmpy = WN_kid0(actual);
    }
    else if (WN_nary_mpy(actual) && WN_kid_count(actual) >= 2)
    {
      narympyI = i;
      narympy = actual;
    }
    else 
    {
      exprI = i;
      expr = actual;
    }

    if (mpy && expr)
    {
      WN_actual(tree, mpyI)= WN_Madd(type, expr, WN_kid0(mpy),
             WN_kid1(mpy));
      tree = WN_NaryDelete(tree, exprI);
      found=  TRUE;
      break;
    }
    else if (nmpy && expr)
    {
      WN_actual(tree, nmpyI)= WN_Nmsub(type, expr, WN_kid0(nmpy),
               WN_kid1(nmpy));
      tree = WN_NaryDelete(tree, exprI);
      found=  TRUE;
      break;
    }
    else if (narympy && expr)
    {
      mpy=  WN_kid0(narympy);
      narympy=  WN_NaryDelete(narympy, 0);

      WN_actual(tree, narympyI)= WN_Madd(type, expr, mpy, narympy);
      tree = WN_NaryDelete(tree, exprI);
      found=  TRUE;
      break;
    }
  }
      }
     /*
      *  There may be madd opportunites at the children
      */
      for(i = 0; i < WN_kid_count(tree); i++)
      {
  WN_actual(tree, i)= lower_madd(block, WN_actual(tree,i), actions);
      }

    }
    
    if (WN_nary_intrinsic(tree))
    {
      tree = WN_NaryToExpr(tree);
    }
  }

  return tree;
}




/* ====================================================================
 *
 * WN *lower_madd(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on expression <tree>,
 * returning lowered expression tree.
 *
 * ==================================================================== */

static WN *lower_madd(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  TYPE_ID  type = WN_rtype(tree);

 /*
  *  look for madd patterns
  */
  switch (WN_operator(tree))
  {
  case OPR_NEG:
    {
      WN  *child = WN_kid0(tree);

      switch(WN_operator(child))
      {
      case OPR_MADD:
  WN_Delete(tree);
  return WN_Nmadd(type, WN_kid0(child), WN_kid1(child), WN_kid2(child));

      case OPR_MSUB:
  WN_Delete(tree);
  return WN_Nmsub(type, WN_kid0(child), WN_kid1(child), WN_kid2(child));

      case OPR_NMADD:
  WN_Delete(tree);
  return WN_Madd(type, WN_kid0(child), WN_kid1(child), WN_kid2(child));

      case OPR_NMSUB:
  WN_Delete(tree);
  return WN_Msub(type, WN_kid0(child), WN_kid1(child), WN_kid2(child));
      }
    }
    break;

  case OPR_ADD:
    {
      WN  *l= WN_kid0(tree);
      WN  *r= WN_kid1(tree);

      if (WN_operator_is(l, OPR_MPY))
      {
  WN_Delete(tree);
  return WN_Madd(type, r, WN_kid0(l), WN_kid1(l));
      }
      else if (WN_operator_is(r, OPR_MPY))
      {
  WN_Delete(tree);
  return WN_Madd(type, l, WN_kid0(r), WN_kid1(r));
      }
    }
    break;

  case OPR_SUB:
    {
      WN  *l= WN_kid0(tree);
      WN  *r= WN_kid1(tree);

      if (WN_operator_is(l, OPR_MPY))
      {
  WN_Delete(tree);
  return WN_Msub(type, r, WN_kid0(l), WN_kid1(l));
      }
      else if (WN_operator_is(r, OPR_MPY))
      {
  WN_Delete(tree);
  return WN_Nmsub(type, l, WN_kid0(r), WN_kid1(r));
      }
    }
    break;

  case OPR_MADD:
    {
      WN  *neg= WN_kid0(tree);

      if (WN_operator_is(neg, OPR_NEG))
      {
  WN  *child= WN_kid0(neg);

  WN_Delete(neg);
  return WN_Msub(type, child, WN_kid1(tree), WN_kid2(tree));
      }
    }
    break;

  case OPR_MSUB:
    {
      WN  *neg= WN_kid0(tree);

      if (WN_operator_is(neg, OPR_NEG))
      {
  WN  *child= WN_kid0(neg);

  WN_Delete(neg);
  return WN_Madd(type, child, WN_kid1(tree), WN_kid2(tree));
      }
    }
    break;

  case OPR_NMADD:
    {
      WN  *neg= WN_kid0(tree);

      if (WN_operator_is(neg, OPR_NEG))
      {
  WN  *child= WN_kid0(neg);

  WN_Delete(neg);
  return WN_Nmsub(type, child, WN_kid1(tree), WN_kid2(tree));
      }
    }
    break;

  case OPR_NMSUB:
    {
      WN  *neg= WN_kid0(tree);

      if (WN_operator_is(neg, OPR_NEG))
      {
  WN  *child= WN_kid0(neg);

  WN_Delete(neg);
  return WN_Nmadd(type, child, WN_kid1(tree), WN_kid2(tree));
      }
    }
    break;
  }

  return tree;
}




/* ====================================================================
 *
 * WN *lower_tree_height(WN *block, WN *wn, LOWER_ACTIONS actions)
 *
 *  Reassociate binary commutative operator at this level 
 *  (ie no recursion)
 *
 * ==================================================================== */

static WN *lower_tree_height(WN *block, WN *wn, LOWER_ACTIONS actions)
{
  OPCODE  opcode = WN_opcode(wn);
  WN    *l, *r;

  if (NotAction(LOWER_TREEHEIGHT))
    return wn;

  Is_True(WN_is_commutative(wn),("expected commutative op"));

  l = WN_kid0(wn);
  r = WN_kid1(wn);

 /*
  *  do not transform an already balanced tree
  */
  if (opcode == WN_opcode(l) ^ opcode == WN_opcode(r))

  {
    WN *wn1 =  (opcode == WN_opcode(l)) ? l : r;
    WN *wn1X = (opcode == WN_opcode(l)) ? r : l;

    WN *wn1_l = WN_kid0(wn1);
    WN *wn1_r = WN_kid1(wn1);

    if (opcode == WN_opcode(wn1_l) ^ opcode == WN_opcode(wn1_r))
    {

      WN *wn2 =  (opcode == WN_opcode(wn1_l)) ? wn1_l : wn1_r;
      WN *wn2X = (opcode == WN_opcode(wn1_l)) ? wn1_r : wn1_l;
     /*
      *  rearrange tree
      */
      WN_kid0(wn1) =  wn2X;
      WN_kid1(wn1) =  wn1X;

      WN_kid0(wn) = wn2;
      WN_kid1(wn) = wn1;

      if (traceTreeHeight)
      {
  DevWarn("lower_tree_height: trace (%s) has been reassociated (line %d)",
          OPCODE_name(WN_opcode(wn)), Srcpos_To_Line(current_srcpos));
      }
    }
  }
  return wn;
}


#ifdef KEY
/* ====================================================================
 *
 * WN *sum_madd_segments(WN **segments, INT low, INT high)
 *
 *  Sum the MADD chains.
 *
 * ==================================================================== */
static WN *sum_madd_segments(WN **segments, INT low, INT high)
{
  WN *tree;

  if (high == low) {
    tree = segments[high];
  } else if (high - low == 1) {
    WN *opnd0 = segments[low];
    WN *opnd1 = segments[high];
    tree = WN_Add(WN_rtype(opnd0), opnd0, opnd1);
  } else {
    INT mid = (high + low) / 2;
    WN *sum0 = sum_madd_segments(segments, low, mid);
    WN *sum1 = sum_madd_segments(segments, mid + 1, high);
    tree = WN_Add(WN_rtype(sum0), sum0, sum1);
  }
  return tree;
}


/* ====================================================================
 *
 * WN *lower_one_madd_tree_height(WN *block, WN *wn)
 *
 *  Reassociate MADD operator at this level 
 *  (ie no recursion)
 *
 * ==================================================================== */

static WN *lower_one_madd_tree_height(WN *block, WN *wn)
{
  // Tree is a MADD and potentially starts a MADD chain.
  Is_True(WN_operator(wn) == OPR_MADD,
    ("lower_one_madd_tree_height: MADD operator not found"));

  if (OPT_Madd_Height <= 0)
    return wn;

  // Count the MADDs in the chain.
  INT madd_count = 0;
  WN *this_wn;
  for (this_wn = wn;
       WN_operator(this_wn) == OPR_MADD;
       this_wn = WN_kid0(this_wn)) {
    madd_count++;
  }

  if (madd_count <= OPT_Madd_Height)
    return wn;

  INT segment_count = madd_count / OPT_Madd_Height + 1;
  WN **segments = (WN **) alloca(segment_count * sizeof(WN *));
  memset(segments, 0, segment_count * sizeof(WN *));

  // Split up the MADD chain.
  INT count = 0;
  INT idx = 0;
  WN *prev_wn = NULL;
  this_wn = wn;
  while (WN_operator(this_wn) == OPR_MADD) {
    WN *next_wn = WN_kid0(this_wn);
    count++;
    if (count > OPT_Madd_Height) {
      WN *mpy = WN_Mpy(WN_rtype(this_wn), WN_kid1(this_wn), WN_kid2(this_wn));
      WN_kid0(prev_wn) = mpy;
      WN_Delete(this_wn);
      count = 0;
    } else if (count == 1) {
      segments[idx++] = this_wn;
    }
    prev_wn = this_wn;
    this_wn = next_wn;
  }

  // If the last MADD in the chain was converted to a MPY, then its kid0 still
  // needs to be summed.
  if (count == 0) {
    segments[idx++] = this_wn;
  }

  // Add the partial sums.
  WN *tree = sum_madd_segments(segments, 0, idx - 1);
  return tree;
}


/* ====================================================================
 *
 * WN *lower_madd_tree_height(WN *block, WN *tree)
 *
 *  Traverse tree to find subtrees that are MADD chains.  Lower the height
 *  of these chains.
 *
 * ==================================================================== */

static void lower_madd_tree_height(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  if (NotAction(LOWER_TREEHEIGHT))
    return;

  if (WN_operator(tree) != OPR_MADD) {
    INT16 i;
    for (i = 0; i < WN_kid_count(tree); i++) {
      WN *kid = WN_kid(tree, i);
      if (WN_operator(kid) == OPR_MADD) {
  // Kid potentially begins a MADD chain.  Reduce its height.
  WN_kid(tree, i) = lower_one_madd_tree_height(block, kid);
      } else {
  lower_madd_tree_height(block, kid, actions);
      }
    }
  }
}
#endif


/* ====================================================================
 *
 * WN *lower_recip(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * WN *lower_rsqrt(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * lower recip/rsqrt based on ISA flags
 *
 * Quad is lowered unconditionally, as there is no runtime support
 * (kludge alert: turn off simplifer when creating quad divide !!)
 *
 * ==================================================================== */

static WN *lower_recip(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  TYPE_ID  type = WN_rtype(tree);
  WN    *kid0 = WN_kid0(tree);
  WN    *div =  NULL;

  Is_True(WN_operator_is(tree, OPR_RECIP), ("expected recip"));
  Is_True(MTYPE_float(type), ("expected float type"));


#ifndef KEY
  if (Recip_Allowed == FALSE)
#else
  if (Recip_Allowed == FALSE && Div_Split_Allowed == FALSE)
#endif
  {
    div = WN_Div(type, WN_Floatconst(type, 1.0), kid0);
  }
  else if (Action(LOWER_QUAD) && MTYPE_is_quad(type))
  {
    BOOL  simp= WN_Simplifier_Enable(FALSE);

    div = WN_Div(type, WN_Floatconst(type, 1.0), kid0);

    WN_Simplifier_Enable(simp);
  }

  if (div)
  {
    WN_Delete(tree);
    return div;
  }

  return tree;
}


static WN *lower_rsqrt(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  TYPE_ID  type = WN_rtype(tree);

  Is_True(WN_operator_is(tree, OPR_RSQRT), ("expected rsqrt"));
  Is_True(MTYPE_float(WN_rtype(tree)), ("expected float type"));

#ifdef TARG_X8664 // bug 6123
  if (Rsqrt_Allowed == 0 ||
      // x86-64 rsqrt supports single-precision only.
      !(type == MTYPE_F4 || type == MTYPE_V16F4) ||
      !Is_Target_SSE2())
#else
  if (Rsqrt_Allowed == FALSE || MTYPE_is_quad(type))
#endif
  {
    WN  *div;

    div = WN_Div(type, WN_Floatconst(type, 1.0), WN_Sqrt(type, WN_kid0(tree)));
    WN_Delete(tree);

    return div;
  }

#ifdef TARG_X8664
  if (Rsqrt_Allowed == 1) {
    /* Rsqrt followed by one iteration of Newton-Raphson to refine the result:
   a = rsqrt(x)      ;; get approximate rsqrt(x)
   b = x * a
   c = b * a         ;; c has error E
   d = c - 3.0
   e = d * a
   f = e * (-0.5)
       f now has 1/sqrt(x).  Bug 6123.

       Change the operator from RSQRT to ATOMIC_RSQRT to prevent infinite
       recursion.
     */
    WN_set_operator(tree, OPR_ATOMIC_RSQRT);
    LEAF x = Make_Leaf(block, WN_kid0(tree), type);
    WN_kid0(tree) = Load_Leaf(x);
    PREG_NUM a = AssignExpr(block, tree, type);
    WN *b = WN_Mpy(type, Load_Leaf(x), WN_LdidPreg(type, a));
    WN *c = WN_Mpy(type, b, WN_LdidPreg(type, a));
    WN *d = WN_Sub(type, c, WN_Floatconst(type, 3.0));
    WN *e = WN_Mpy(type, d, WN_LdidPreg(type, a));
    WN *f = WN_Mpy(type, e, WN_Floatconst(type, -0.5));
    return f;
  }
#endif

  return tree;
}


/* IPA can mark variables gp-relative, even if they are not allocated
 * in this source file. To deal with them, we explicitly check if a 
 * variable is gp-relative.
 */
static BOOL Symbol_Is_Base_Register (ST *sym)
{
  return ((ST_class(sym) == CLASS_BLOCK && STB_is_basereg(sym)) ||
          (ST_class(sym) == CLASS_VAR && ST_gprel(sym)));
}



/* ====================================================================
 *
 * WN *lower_split_sym_addrs(WN *tree, INT64 offset, LOWER_ACTIONS actions)
 *
 * Split symbols into base/offset depending on type/class etc.
 *
 *
 * ==================================================================== */

static WN *lower_split_sym_addrs(WN *tree, INT64 offset, LOWER_ACTIONS actions)
{
  ST *sym  = WN_st(tree);
  ST *base = ST_base(sym);
  INT64 newoffset = 0;

#ifdef TARG_NVISA
  // want to preserve original symbol
  return NULL;
#endif

  if (traceSplitSymOff)
    return NULL;

  switch(ST_class(sym))
  {
  case CLASS_CONST:
   /*
    *  for non_shared, non-gprelative constants we want to expose the
    *  base for RVI
    */
    if (PIC_NONSHARED && (ST_size(sym) > Max_Sdata_Elt_Size))
    {
      Allocate_Object(sym);
      Base_Symbol_And_Offset_For_Addressing(sym, offset, &base, &newoffset);
      return lower_base_reference(tree, base, newoffset, actions);
    }
    return NULL;

  case CLASS_PREG:
    return NULL;

  case CLASS_FUNC:
    if (PIC_SHARED && ST_is_preemptible(sym))
    {
      return NULL;
    }
    break;

  case CLASS_BLOCK:
  case CLASS_VAR:
    Base_Symbol_And_Offset_For_Addressing(sym, offset, &base, &newoffset);

    if (ST_is_uplevelTemp(sym))
    {
      return NULL;
    }
    if (ST_assigned_to_dedicated_preg(sym)) 
    {
  return NULL;
    }

   /*
    *  We cannot keep lowering LDA's or we will forever recurse
    */
    if (WN_operator_is(tree, OPR_LDA) && base == sym) 
      return NULL;

    switch(ST_sclass(sym))
    {
    case SCLASS_REG:
    case SCLASS_TEXT:
      return NULL;

    case SCLASS_FORMAL_REF: 
     /*
      *  If we expand the base, we will lose the FORMAL_REF'ness of the ST
      *  and will not be able to dereference it later
      */
      if (base != sym)
      {
  base =  sym;
      }
      break;

    case SCLASS_FORMAL:
     /*
      *  Do not allocate Formal_Arg_StkSeg, as it is not correct yet !!
      *  (it will eventually be .sp + <offset> )
      */
      return NULL;

    case SCLASS_AUTO:
      /* 
       * For now, never split a stack variable.
       * We only see stack variables with bases when we do regions,
       * and for now we want to keep those cases looking like the
       * regular cases for correctness.  But at a later date we should
       * try to split these in the region case, as that may allow a
       * base register to be allocated for things like FP+32000,
       * so we could then get 1-instruction accesses to large offsets.
       * But that requires a cgexp change too, so wait on it.
       */
#if 0
      if (Uses_Small_Offset(sym, offset))
        break;
#endif
      return NULL;

    case SCLASS_EXTERN:
     /*
      * okay to split these. A dlopen() can never redefine these (unless
      * they are weak)
      */
      if (ST_is_weak_symbol(sym))
  return NULL;
      break;

    case SCLASS_COMMON:
     /*
      * commons may be preempted by a dlopen() (if the new definition is
      * initialized) see wilson/lillian for details of this rather obscure
      * point suneel says not to worry about this though, so please direct
      * bugs to him
      *
      * In the new symbol table, we need to also split common symbols
      * here. The reason is that now both members and the common itself
      * are SCLASS_COMMON, whereas before they used to be SCLASS_BASED.
      * We no longer have to worry about the ST_Full that was present in
      * the old symbol table
      */
      sym = base;
      break;

    case SCLASS_PSTATIC:
    case SCLASS_UGLOBAL:
    case SCLASS_DGLOBAL:
    case SCLASS_FSTATIC:
     /*
      *  okay to split these. We will never split incorrectly as
      *  Base_Symbol_And_Offset_For_Addressing() will not split a preemptible
      *  symbol
      */
      break;

    default:
      return NULL;
    }

#ifdef TARG_SL2 
    if(ST_gprel(sym) || WN_is_internal_mem_ofst(tree))
#else 
    if (ST_gprel(sym))
#endif    
    {
      return NULL;
    }
    tree = lower_base_reference(tree, base, newoffset, actions);
    return tree;

  default:
    return NULL;
  }

  return NULL;
}




/* ====================================================================
 *
 * WN *lower_uplevel_reference(WN *tree, INT64 offset, LOWER_ACTIONS actions)
 *
 * Split uplevel symbols into indirect of slink
 *
 * ==================================================================== */

static WN *lower_uplevel_reference(WN *tree, INT64 offset,
           LOWER_ACTIONS actions)
{
  ST  *sym = WN_st(tree);
  ST  *base;
  INT64 newoffset;

  Is_True(ST_is_uplevelTemp(sym), ("expected uplevel %s",ST_name(sym)));

  Base_Symbol_And_Offset_For_Addressing(sym, offset, &base, &newoffset);
  base = Find_Slink_For_ST(sym);

  tree = lower_dereference(tree, newoffset, base, 0, actions);
  return tree;
}




/* ====================================================================
 *
 * WN *lower_formal_ref(WN *tree, INT64 offset, ST *base,
 *                      LOWER_ACTIONS actions)
 *
 * lower an SCLASS_FORMAL_REF into a dereference of the new base.
 * or
 * lower an SCLASS_FORMAL into a preg, already computed
 *
 * ==================================================================== */
static WN *lower_formal_ref(WN *tree, INT64 offset, ST *base,
          LOWER_ACTIONS actions)
{
  PREG_NUM  preg;

  switch(ST_sclass(base))
  {
  case SCLASS_FORMAL_REF:
    base = Get_ST_formal_ref_base(base);
    if (preg = Get_ST_formal_preg_num(base))
    {
      Is_True((ST_has_nested_ref(WN_st(tree))==FALSE),
        ("lower_formal_ref: cannot use preg for nested ref %s",
         ST_name(WN_st(tree))));
      base = MTYPE_To_PREG(TY_mtype(Ty_Table[ST_type(base)]));
    }
    return lower_dereference(tree, offset, base, preg, actions);

  case SCLASS_FORMAL:
    if (preg= Get_ST_formal_preg_num(base))
    {
      base = MTYPE_To_PREG(TY_mtype(Ty_Table[ST_type(base)]));
      tree = lower_base_register(tree, base, preg, actions);
      return tree;
    }
    break;
  }
  return NULL;
}




/* ====================================================================
 *
 * WN *lower_base_register(WN *tree, ST *base, INT64 offset,
 *                         LOWER_ACTIONS actions)
 *
 * common routine for lowering to a base register for LDA, LDID, STID
 * into using a new base/offset
 *
 * ==================================================================== */

static WN *lower_base_register(WN *tree, ST *base, INT64 offset,
             LOWER_ACTIONS actions)
{
  Is_True(WN_st(tree) != base, ("lower_base_register(): possible recursion"));

  switch (WN_operator(tree))
  {
  case OPR_LDA:
   /*
    * Convert
    *     LDA (offset) <sym> into
    *
    *     LDA (offset) <base>
    */
    WN_st_idx(tree) = ST_st_idx(base);
    WN_lda_offset(tree) = mem_offset_hi(offset);
    return tree;

  case OPR_LDID:
   /*
    * Convert
    *   LDID (offset> <sym>  into
    *     LDID (offset <base>
    */
    WN_st_idx(tree) = ST_st_idx(base);
    WN_load_offset(tree) = offset;
    break;

  case OPR_STID:
   /*
    * Convert
    *   STID (offset> <sym>  into
    *     STID (offset <base>
    */
    WN_st_idx(tree) = ST_st_idx(base);
    WN_store_offset(tree) = offset;
    break;

  default:
    Is_True(FALSE,("expected lda/ldid/stid"));
    return NULL;
  }

  return tree;
}


// replace the type of LDID/STID nodes with non-zero field_id to the type
// of the field
static void
lower_field_id (WN* tree)
{
  OPERATOR opr = WN_operator(tree);

  Is_True (opr == OPR_LDID || opr == OPR_STID || opr == OPR_MLOAD ||
     opr == OPR_MSTORE, ("expecting LDID or STID nodes"));

  if (WN_field_id (tree) == 0)
    return;

  BOOL is_ptr_type = (opr == OPR_MLOAD || opr == OPR_MSTORE);

  TY_IDX ty_idx = is_ptr_type ? TY_pointed (WN_ty (tree)) : WN_ty (tree);

  Is_True (TY_kind (ty_idx) == KIND_STRUCT,
     ("non-zero field id must have KIND_STRUCT"));

  UINT field_id = 0;
  FLD_HANDLE fld = FLD_get_to_field (ty_idx, WN_field_id (tree), field_id); 
  
  Is_True (! fld.Is_Null (), ("invalid bit-field ID for %s",
            OPERATOR_name(opr)));

  WN_set_ty (tree, (is_ptr_type ? Make_Pointer_Type (FLD_type (fld)) :
        FLD_type (fld)));
  WN_set_field_id (tree, 0);
  return;
} // lower_field_id


/* ====================================================================
 *
 * WN *lower_base_reference(WN *tree, ST *base, INT64 offset,
 *                          LOWER_ACTIONS actions)
 *
 * common routine for lowering to a base reference for LDA, LDID, STID
 *
 * ==================================================================== */

static WN *lower_base_reference(WN *tree, ST *base, INT64 offset,
        LOWER_ACTIONS actions)
{
  WN    *addr, *wn;

  switch (WN_operator(tree))
  {
  case OPR_LDA:
   /*
    * Convert
    *     LDA (offset) <sym>
    * 
    * (lo > 0)
    *       LDA (hi) <base>
    *       CONST (lo)
    *     ADD
    *
    * (offset >= 2GB)
    *       LDA (0) <base>
    *       CONST (offset)
    *     ADD
    */

    WN_st_idx(tree) = ST_st_idx(base);

    if (mem_offset_2GB(offset))
    {
      addr =  WN_Add(Pointer_type, tree, WN_Intconst(Pointer_type, offset));
      WN_lda_offset(tree)=  0;
      return addr;
    }
    else if (Action(LOWER_SPLIT_CONST_OFFSETS)  &&
       mem_offset_lo(offset))
    {
     /* turn off simplifier else ADD may be removed */
      BOOL  simp=        WN_Simplifier_Enable(FALSE);

      addr =  WN_Add(Pointer_type,
           tree,
           WN_Intconst(Pointer_type, mem_offset_lo(offset)));
      WN_lda_offset(tree)=  mem_offset_hi(offset);

      WN_Simplifier_Enable(simp);
      return addr;
    }
    WN_lda_offset(tree)=  offset;
    return tree;

  case OPR_LDID:
  case OPR_STID:
  case OPR_LDBITS:
  case OPR_STBITS:
   /*
    * Process the common address
    *
    * if (offset > 2GB)
    *      LDA (hi) <sym>
    *    CONST offset
    *    ADD
    *
    * if (offset > 16K)
    *      LDA (hi) <sym>
    *    ILOAD | ISTORE (lo)
    */
    addr =  WN_Lda(Pointer_type, 0, base);

    if (mem_offset_2GB(offset))
    {
      addr =  WN_Add(Pointer_type, addr, WN_Intconst(Pointer_type, offset));
      offset =  0;
    }
    if (Action(LOWER_SPLIT_CONST_OFFSETS) && mem_offset_must_be_split(offset))
    {
      WN_lda_offset(addr) = mem_offset_hi(offset);
      offset =      mem_offset_lo(offset);
    }
    break;

  default:
    Is_True(FALSE,("expected lda/ldid/stid"));
    return NULL;
  } 

  switch (WN_operator(tree))
  {
  case OPR_LDID:
   /*
    * Convert (LDID () <sym>) into
    *      LDA (0) <base>
    *    ILOAD (offset+ofst)
    */
    if (WN_field_id (tree) != 0)
      lower_field_id (tree);
    
    wn =  WN_RIload(WN_rtype(tree), WN_desc(tree), offset, WN_ty(tree),
        addr);
    break;
  case OPR_LDBITS:
    wn =  WN_RIload(WN_rtype(tree), WN_desc(tree), offset, WN_ty(tree),
        addr);
    WN_set_operator(wn, OPR_ILDBITS);
    WN_set_bit_offset_size(wn, WN_bit_offset(tree), WN_bit_size(tree));
    break;

  case OPR_STID:
   /*
    * Convert   STID (offset) <sym>
    *  into
    *       LDA (0) <base>
    *     ISTORE (offs+ofst)
    */
    if (WN_field_id (tree) != 0)
      lower_field_id (tree);
    
    wn =  WN_Istore(WN_desc(tree),
        offset,
        // The resulting TY is not an f90 pointer
        // type because we're converting an STID.
        // In the case where this STID
        // came about because of conversion of
        // ISTORE(something) in the past, and that
        // ISTORE was through an f90 pointer, the
        // destination of the STID will have the
        // F90_TARGET attribute, and we're OK.
        Make_Pointer_Type(WN_ty(tree)),
        addr,
        WN_kid0(tree));
    WN_Set_Linenum(wn, WN_Get_Linenum(tree));
    break;
  case OPR_STBITS:
    wn =  WN_Istore(WN_desc(tree),
        offset,
        Make_Pointer_Type(WN_ty(tree)),
        addr,
        WN_kid0(tree));
    WN_set_operator(wn, OPR_ISTBITS);
    WN_set_bit_offset_size(wn, WN_bit_offset(tree), WN_bit_size(tree));
    WN_Set_Linenum(wn, WN_Get_Linenum(tree));
    break;
  } 

  lower_copy_maps(tree, wn, actions);
  WN_Delete(tree);

  return wn;
}



/* ====================================================================
 *
 * WN *lower_dereference(WN *tree, INT64 offset, ST *base,
 *                       PREG_NUM *preg, LOWER_ACTIONS actions)
 *
 * Perform address dereferencing for SCLASS_FORMAL_REF and uplevel variable
 * Caller is responsible for any further lowering.
 *
 *  For uplevel references the idea is to replace the st with a reference
 *  of the slink_sym at level ST_level(st) + 1
 *  This will only work recursively
 *  (ie. that sym must be referenced via the next level's scope
 *
 * ==================================================================== */
static WN
*lower_dereference(WN *tree, INT64 offset, ST *base, PREG_NUM preg,
       LOWER_ACTIONS actions)
{
  WN   *addr, *deref;
  TY_IDX  addr_ty;

  switch (WN_operator(tree))
  {
  case OPR_LDA:
   /*
    * Convert uplevel reference
    *     LDA (offset) <sym>
    *  into 
    *       LDID (0) <base>
    *       CONST (offset)    
    *     ADD
    */
    deref = WN_Ldid(Pointer_type, preg, base, WN_ty(tree));
    addr =  WN_Add(Pointer_type, deref, WN_Intconst(Pointer_type, offset));
    return addr;

  case OPR_LDID:
  case OPR_STID:
   /*
    * Process the common address
    *
    */
    addr_ty = Make_Pointer_Type(WN_ty(tree));
    addr =  WN_Ldid(Pointer_type, preg, base, addr_ty);

    if (mem_offset_2GB(offset))
    {
      addr =  WN_Add(Pointer_type, addr, WN_Intconst(Pointer_type, offset));
      offset =  0;
    }
    break;

  default:
    Is_True(FALSE,("expected lda/ldid/stid"));
    return NULL;
  } 

  switch (WN_operator(tree))
  {
  case OPR_LDID:
   /*
    * Convert uplevel reference  LDID () <sym>
    * into
    *      LDID (preg) <base>
    *    ILOAD (offset)
    */
    deref = WN_RIload(WN_rtype(tree), WN_desc(tree), offset, WN_ty(tree), addr,
              WN_field_id(tree));
    break;

  case OPR_STID:
   /*
    * Convert  STID () <sym>
    * into
    *       LDID (preg) <base> 
    *     ISTORE (offset)          
    */
    deref = WN_Istore(WN_desc(tree), offset, addr_ty, addr, WN_kid0(tree)
#ifdef KEY // bug 14036
                      ,WN_field_id(tree)
#endif
                     );
    WN_Set_Linenum(deref, WN_Get_Linenum(tree));
    break;
  } 

  lower_copy_maps(tree, deref, actions);
  WN_Delete(tree);

  return deref;
}

/* ====================================================================
 *
 * WN *lower_return_ldid(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on 'LDID Return_Val_Preg'
 * nodes returning a lowered 'LDID normal_PREG' node.
 *
 * ==================================================================== */

static WN *lower_return_ldid(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  TY_IDX   ty_idx  = WN_ty(tree);
  TY&      ty      = Ty_Table[ty_idx];
  TYPE_ID  mtype   = TY_mtype (ty);
#ifdef TARG_X8664
  BOOL sseregparm  = FALSE; // sseregparm attribute on called function?
#endif

  Is_True((WN_operator(tree) == OPR_LDID),
    ("expected LDID node, not %s", OPCODE_name(WN_opcode(tree))));

#ifdef TARG_X8664
  if (Is_Target_32bit())
  {
    // Determine if the called function has the "sseregparm" attribute
    WN * call = WN_last(block);
    if (call && WN_operator(call) == OPR_CALL)
    {
      TY_IDX pu_ty = PU_prototype(Pu_Table[ST_pu(WN_st(call))]);
      sseregparm = TY_has_sseregister_parm (pu_ty);
    }
  }
#endif

  switch (mtype) {

    case MTYPE_I8:
    case MTYPE_U8:
#if defined(TARG_X8664)
      WN_st_idx(tree) = ST_st_idx(MTYPE_To_PREG(mtype));
#else
      WN_st_idx(tree) = ST_st_idx(Int64_Preg);
#endif
      WN_load_offset(tree) = First_Int_Preg_Return_Offset;
#ifdef TARG_NVISA
      // int64 are separate register class, so use unique preg num
      WN_load_offset(tree) = First_Int64_Preg_Return_Offset;
#endif
      return tree;

    case MTYPE_I1:
    case MTYPE_I2:
    case MTYPE_I4:
    case MTYPE_U1:
    case MTYPE_U2:
    case MTYPE_U4:
#ifdef TARG_NVISA
      WN_st_idx(tree) = ST_st_idx(Int32_Preg);
#else
      WN_st_idx(tree) = ST_st_idx(Int_Preg);
#endif
      WN_load_offset(tree) = First_Int_Preg_Return_Offset;
      return tree;

    case MTYPE_F4:
    case MTYPE_F8:
    case MTYPE_F10:
    case MTYPE_FQ:
    case MTYPE_C4:
    case MTYPE_C8:
    case MTYPE_C10:
    case MTYPE_CQ:
#if defined(TARG_SL) && defined(EMULATE_FLOAT_POINT)
#if defined(TRACE_LOWER) 
      fprintf(TFile, "wn_lower.cxx: lower_return_ldid %d at line %d\n", mtype, __LINE__);
#endif
      WN_st_idx(tree) = (mtype == MTYPE_F8) ? ST_st_idx(Int64_Preg) : ST_st_idx(Int32_Preg);
      WN_load_offset(tree) = First_Int_Preg_Return_Offset; 
#elif defined(TARG_NVISA)
      if (mtype == MTYPE_F8) {
  // doubles are separate register class, so use unique preg num
        WN_st_idx(tree) = ST_st_idx(Float64_Preg);
        WN_load_offset(tree) = First_Float64_Preg_Return_Offset;
      } else {
        WN_st_idx(tree) = ST_st_idx(Float32_Preg);
        WN_load_offset(tree) = First_Float_Preg_Return_Offset;
      }
#else
      WN_st_idx(tree) = ST_st_idx(Float_Preg);
      WN_load_offset(tree) = First_Float_Preg_Return_Offset;
#endif
#ifdef TARG_X8664 // MTYPE_C4 is returned in one SSE register
      if (Is_Target_64bit() && mtype == MTYPE_C4) {
  WN_set_rtype(tree, MTYPE_F8);
  WN_set_desc(tree, MTYPE_F8);
  WN_set_ty(tree, MTYPE_To_TY(MTYPE_F8));
      }

      if( Is_Target_32bit() && ! sseregparm && ! MTYPE_is_complex(mtype) ||
            MTYPE_is_quad(mtype) ){
  WN_st_idx(tree) = ST_st_idx( MTYPE_To_PREG(mtype) );
  WN_load_offset(tree) = First_X87_Preg_Return_Offset;
      }
#endif
      return tree;

#if defined(TARG_X8664) || defined(VECTOR_MTYPES)
    case MTYPE_V16I1:
    case MTYPE_V16I2:
    case MTYPE_V16I4:
    case MTYPE_V16I8:
    case MTYPE_V16F4:
    case MTYPE_V16F8:
    case MTYPE_V8I1:
    case MTYPE_V8I2:
    case MTYPE_V8I4:
#ifdef TARG_X8664
    case MTYPE_M8I1:
    case MTYPE_M8I2:
    case MTYPE_M8I4:
    case MTYPE_M8F4:
#endif
      WN_st_idx(tree) = ST_st_idx(MTYPE_To_PREG (mtype));
      WN_load_offset(tree) = First_Float_Preg_Return_Offset;
      return tree;
#endif // TARG_X8664 || VECTOR_MTYPES

    case MTYPE_M:
      Fail_FmtAssertion ("MLDID of Return_Val_Preg not allowed in middle"
       " of expression");
      /*NOTREACHED*/
    default:
      Fail_FmtAssertion ("Unexpected type in lower_return_ldid");
      /*NOTREACHED*/
  }
}

static TY_IDX
get_field_type (TY_IDX struct_type, UINT field_id)
{
  Is_True (TY_kind (struct_type) == KIND_STRUCT, ("expecting KIND_STRUCT"));
  UINT cur_field_id = 0;
  FLD_HANDLE fld = FLD_get_to_field (struct_type, field_id, cur_field_id);
  Is_True (! fld.Is_Null(), ("Invalid field id %d for type 0x%x",
           field_id, struct_type));
  return FLD_type (fld);
}

/* ====================================================================
 *
 * WN *lower_mldid(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on MLDID nodes returning
 * an equivalent MLOAD node.  Note that MLDID's of Return_Val_Preg are
 * not lowered here.  They are handled by lower_return_ldid.
 *
 * ==================================================================== */

static WN *lower_mldid(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  TY_IDX ty_idx  = WN_ty(tree);
  TY_IDX pty_idx;
  UINT64 size    = WN_field_id(tree) == 0 ?
      TY_size(ty_idx) :
      TY_size(get_field_type (ty_idx, WN_field_id (tree)));
  WN*    wn;
  WN*    awn;
  WN*    swn;

  Is_True((WN_opcode(tree) == OPC_MMLDID),
    ("expected mldid node, not %s", OPCODE_name(WN_opcode(tree))));

  pty_idx = Make_Pointer_Type (ty_idx, FALSE);

  swn = WN_CreateIntconst(OPC_U4INTCONST, size);
  awn = WN_CreateLda(OPR_LDA, Pointer_Mtype, MTYPE_V, WN_load_offset(tree),
         pty_idx, WN_st(tree));
  wn  = WN_CreateMload(0, pty_idx, awn, swn);
  WN_set_field_id(wn, WN_field_id(tree));
  wn  = lower_expr(block, wn, actions);

  WN_Delete(tree);
  return wn;
}

/* ====================================================================
 *
 * WN *lower_miload(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on MILOAD nodes returning
 * an equivalent MLOAD node.
 *
 * ==================================================================== */

static WN *lower_miload(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  TY_IDX ty_idx  = WN_field_id (tree) == 0 ?
    WN_ty(tree) :
    get_field_type (WN_ty (tree), WN_field_id (tree));
  TY_IDX pty_idx = WN_load_addr_ty(tree);
  UINT64 size    = TY_size(Ty_Table[ty_idx]);
  WN*    wn;
  WN*    swn;

  Is_True((WN_opcode(tree) == OPC_MMILOAD),
    ("expected miload node, not %s", OPCODE_name(WN_opcode(tree))));

  swn = WN_CreateIntconst (OPC_U4INTCONST, size);
  wn  = WN_CreateMload (WN_offset(tree), pty_idx, WN_kid0(tree), swn);
  WN_set_field_id(wn, WN_field_id(tree));
  set_original_wn (wn, tree);
  wn  = lower_expr (block, wn, actions);

  WN_Delete (tree);
  return wn;
}

/* ====================================================================
 *
 * lower_load_bits
 *
 * lower LDBITS and ILDBITS into shifts
 *
 * ==================================================================== */
static WN*
lower_load_bits (WN* block, WN* wn, LOWER_ACTIONS actions)
{
  Is_True (WN_operator (wn) == OPR_LDBITS || WN_operator (wn) == OPR_ILDBITS,
     ("expected LDBITS or ILDBITS, not %s",
      OPERATOR_name(WN_operator(wn))));

  TYPE_ID rtype = WN_rtype (wn);
  TYPE_ID desc = WN_desc (wn);
  INT delta = MTYPE_bit_size(rtype) - MTYPE_bit_size(desc);
  if (delta < 0) {
    rtype = Mtype_TransferSize (desc, rtype);
    delta = 0;
  }

  WN* tree = wn;

  INT bit_size = WN_bit_size (wn);
  INT bit_ofst = Target_Byte_Sex == BIG_ENDIAN ?
    WN_bit_offset (wn) :
    MTYPE_bit_size(desc) - bit_size - WN_bit_offset (wn);
  BOOL bits_signed = MTYPE_signed(rtype);
  
  if (bit_ofst == 0)
    bit_size += delta;
  else {
    bit_ofst += delta;
    if (bits_signed)
      tree = WN_Shl (rtype, tree, WN_Intconst (MTYPE_I4, bit_ofst));
    else {
      INT shift_count = 64 - (MTYPE_bit_size(rtype) - bit_ofst);
      mUINT64 mask = (~(mUINT64)0) >> shift_count;
      tree = WN_Band (rtype, tree,
          WN_Intconst (Mtype_TransferSign (MTYPE_U4, rtype),
           mask));
      bit_size += bit_ofst;
    }
  }

  INT right_shift = MTYPE_bit_size(rtype) - bit_size;

  if (right_shift > 0) {
    OPERATOR opr = bits_signed ? OPR_ASHR : OPR_LSHR;
    tree = WN_Binary (opr, rtype, tree, WN_Intconst (MTYPE_I4, right_shift));
  }

  TYPE_ID orig_rtype = WN_rtype (wn);
  
  WN_set_rtype (wn, rtype);
  WN_set_bit_offset_size (wn, 0, 0);
  WN_set_operator (wn, WN_operator (wn) == OPR_LDBITS ? OPR_LDID : OPR_ILOAD);

  if (rtype != orig_rtype)
    tree = WN_Type_Conversion (tree, orig_rtype);

  return lower_expr (block, tree, actions);
} // lower_load_bits


/* ====================================================================
 *
 * lower_store_bits
 *
 * lower STBITS and ISTBITS
 *
 * ==================================================================== */
static WN*
lower_store_bits (WN* block, WN* wn, LOWER_ACTIONS actions)
{
  Is_True (WN_operator (wn) == OPR_STBITS || WN_operator (wn) == OPR_ISTBITS,
     ("expected STBITS or ISTBITS, not %s",
      OPERATOR_name (WN_operator (wn))));

  INT bit_size = WN_bit_size (wn);
  INT bit_ofst = WN_bit_offset (wn); 

  WN* orig_value;

  if (WN_operator (wn) == OPR_ISTBITS) {
    WN* load_address = lower_copy_tree (WN_kid1 (wn), actions);
    orig_value = WN_Iload (Mtype_TransferSign (MTYPE_U4, WN_desc (wn)),
         WN_offset (wn), WN_ty (wn), load_address, 0);
  } else 
    orig_value = WN_Ldid (Mtype_TransferSign (MTYPE_U4, WN_desc (wn)),
        WN_offset (wn), WN_st_idx (wn), WN_ty (wn), 0);
  
  TYPE_ID cmp_type = WN_rtype (orig_value);
  
  INT shift = Target_Byte_Sex == BIG_ENDIAN ?
      MTYPE_bit_size (WN_desc (wn)) - bit_ofst - bit_size :
      bit_ofst;
  mUINT64 mask = ~((~((mUINT64)0) >> (64 - bit_size)) << shift);
  orig_value = WN_Band (cmp_type, orig_value, WN_Intconst (cmp_type, mask));

  WN* new_value = WN_kid0 (wn);

  // check if we need to sign-extend the value
  if (bit_size > MTYPE_bit_size (WN_rtype (new_value)))
    new_value =
      WN_CreateCvtl (OPR_CVTL,
         Mtype_TransferSign (WN_rtype (new_value), cmp_type),
         WN_rtype (new_value),
         MTYPE_bit_size (WN_rtype (new_value)),
         new_value);
  
  // now, truncate to the right bit size for store
  mask = ~((mUINT64)0) >> (64 - bit_size);
  new_value =
    WN_Band (cmp_type, new_value,
       WN_Intconst (Mtype_TransferSize (WN_rtype (new_value), cmp_type),
        mask));

  // move the bits to the right position
  if (shift > 0)
    new_value = WN_Shl (cmp_type, new_value, WN_Intconst (cmp_type, shift));

  // set the value with bitwise or
  new_value = WN_Bior (cmp_type, orig_value, new_value);

  WN_kid0 (wn) = new_value;
  WN_set_bit_offset_size (wn, 0, 0);
  WN_set_operator (wn, WN_operator(wn) == OPR_STBITS ? OPR_STID : OPR_ISTORE);

  return lower_store (block, wn, actions);
} // lower_store_bits

#if 0
/* ====================================================================
 *
 * check_unaligned
 *
 * required_alignment is the natural alignment; offset is the actual offset
 * used in the current access; ty_align is the alignment in the TY of the
 * current access.  Return whether the access is unaligned.
 *
 * ==================================================================== */
static bool check_unaligned(INT required_alignment, INT offset, INT ty_align)
{
  if (required_alignment <= 1)
    return FALSE;
  INT align = ty_align;
  if (offset) {
    INT offset_align = offset % required_alignment;
    if (offset_align)
      align = MIN(align, offset_align);
  }
  return align < required_alignment;
}
#endif


// --------------------------------------------------------------------
// This function mimics FLD_get_to_field from common/com/symtab.cxx,
// but it also computes the offset of <field_id> within <struct_ty_idx>
// We need this because FLD_ofst only gives the offset within the first
// enclosing struct.
// --------------------------------------------------------------------
FLD_HANDLE
FLD_And_Offset_From_Field_Id (TY_IDX  struct_ty_idx, 
                              UINT    field_id, 
                              UINT&   cur_field_id,
                              UINT64& offset)
{
  FLD_ITER fld_iter = Make_fld_iter(TY_fld(struct_ty_idx));
  do {
    FLD_HANDLE fld(fld_iter);
    cur_field_id++;
    if (cur_field_id == field_id) {
      offset += FLD_ofst(fld);
      return fld;
    }
    if (TY_kind(FLD_type(fld)) == KIND_STRUCT &&
        TY_fld(FLD_type(fld)) != FLD_HANDLE()) {
      UINT64 nested_offset = offset + FLD_ofst(fld);
      fld = FLD_And_Offset_From_Field_Id(FLD_type(fld), field_id, 
                                         cur_field_id, nested_offset);
      if (cur_field_id == field_id) {
        offset = nested_offset;
        return fld;
      }
    }
  } while (!FLD_last_field(fld_iter++));
  
  return FLD_HANDLE();
} 

/* ====================================================================
 *
 * lower_bit_field_id
 *
 * The given LDID/STID/ILOAD/ISTORE node has desc type MTYPE_BS.  Lower
 * the node by changing the field_id to bit_offset/bit_size combination.
 * The desc field is changed to reflect the unit in memory to load, and
 * the offset field may need to be updated.
 *
 * ==================================================================== */
static void lower_bit_field_id(WN *wn)
{
  TY_IDX struct_ty_idx;
  TY_IDX ty_idx;
  TYPE_ID rtype;
  OPERATOR opr = WN_operator(wn);
  OPERATOR new_opr;
  BOOL indirect;
  if (opr == OPR_LDID || opr == OPR_STID) {
    ST_IDX st_idx = WN_st_idx(wn);
    struct_ty_idx = WN_ty(wn);
    new_opr = (opr == OPR_LDID) ? OPR_LDBITS : OPR_STBITS;
    indirect = FALSE;
  }
  else {
    if (WN_operator(wn) == OPR_ILOAD) {
      ty_idx = WN_load_addr_ty(wn);
      new_opr = OPR_ILDBITS;
    }
    else {  // ISTORE
      ty_idx = WN_ty(wn);
      new_opr = OPR_ISTBITS;
    }
    Is_True(TY_kind(ty_idx) == KIND_POINTER,
      ("addr ty not pointer type for %s", OPERATOR_name(opr)));
    struct_ty_idx = TY_pointed(ty_idx);
    indirect = TRUE;
  }
  Is_True(TY_kind(struct_ty_idx) == KIND_STRUCT,
    ("struct type not associated with bit-field access for %s", OPERATOR_name(opr)));
  UINT cur_field_id = 0;
  UINT64 field_offset = 0;
  FLD_HANDLE fld = FLD_And_Offset_From_Field_Id(struct_ty_idx, 
                                                WN_field_id(wn),
                                                cur_field_id,
                                                field_offset); 
  Is_True(! fld.Is_Null(),
    ("invalid bit-field ID for %s", OPERATOR_name(opr)));
  TY_IDX fld_ty_idx = FLD_type(fld);
  
  Is_True(FLD_is_bit_field(fld),
    ("non-bit-field associated with bit-field access for  %s", OPERATOR_name(opr)));

  // analyze bit field accessed
  UINT bytes_accessed = TY_size(fld_ty_idx);
  if (OPERATOR_is_store(new_opr))
    rtype = TY_mtype(fld_ty_idx);
  else rtype = WN_rtype(wn);
  INT ofst = field_offset;
#if 0
  BOOL unaligned_field = check_unaligned(bytes_accessed * 8, ofst,
           TY_align(struct_ty_idx));
#endif
  if (ofst >= 0)
    ofst = ofst / bytes_accessed * bytes_accessed;
  else ofst =  (ofst - bytes_accessed + 1) / bytes_accessed * bytes_accessed;
  UINT bsize = FLD_bsize(fld);
  UINT bofst = FLD_bofst(fld) + (field_offset-ofst) * 8;
  if ((bofst + bsize) > (bytes_accessed * 8)) {
#ifdef TARG_X8664
    if (bytes_accessed == MTYPE_byte_size(MTYPE_I8)){ 
#else
    if (bytes_accessed == MTYPE_byte_size(Max_Int_Mtype)){ 
#endif // TARG_X8664
      // can't enlarge; reverse the adjustment
      ofst = field_offset;
      bofst = FLD_bofst(fld);
    }
    else bytes_accessed *= 2;
  }
  WN_load_offset(wn) = WN_load_offset(wn) + ofst; 

  TYPE_ID mtype;
  if ((bsize & 7) == 0 &&        // field size multiple of bytes
#ifdef KEY // bug 11076
      bsize &&           // field size non-zero
#endif
      (bytes_accessed * 8 % bsize) == 0 && // bytes_accessed multiple of bsize
#ifdef TARG_X8664
      (bofst & 7) == 0
#else
      (bofst % bsize) == 0       // bofst multiple of bsize
#endif // TARG_X8664
      ) {
    // bit-field operation not needed; leave operator as previous one
    WN_set_field_id(wn, 0);
    mtype = Mtype_AlignmentClass(bsize >> 3, MTYPE_type_class(rtype));
    WN_set_desc(wn, mtype);
    WN_load_offset(wn) = WN_load_offset(wn) + (bofst >> 3);
  }
  else { // generate lowered-to bit-field operator
#if defined(TARG_MIPS) || defined(TARG_IA32)
    if ((indirect || WN_class(wn) != CLASS_PREG) && 
  bofst % 8 == 0 &&   // bit offset on byte boundary
  compute_offset_alignment(bytes_accessed*8, bofst) >= bsize) {
      // adjust bytes accessed to get rid of the left-shift
      WN_load_offset(wn) = WN_load_offset(wn) + (bofst >> 3);
      bytes_accessed = compute_offset_alignment(bytes_accessed, bofst >> 3);
      bofst = 0;
    }
#endif
    WN_set_operator(wn, new_opr);

#ifdef KEY
    mtype = Mtype_AlignmentClass(bytes_accessed, MTYPE_type_class(rtype));
    Is_True( mtype != MTYPE_UNKNOWN, ("Unknown mtype encountered.") );
    WN_set_desc(wn, mtype);
#else
    WN_set_desc(wn, Mtype_AlignmentClass(bytes_accessed, MTYPE_type_class(rtype)));
#endif // KEY

    if (OPERATOR_is_load(new_opr) && 
  MTYPE_byte_size(WN_rtype(wn)) < bytes_accessed)
      WN_set_rtype(wn, WN_desc(wn));
    WN_set_bit_offset_size(wn, bofst, bsize);
  }

  // fix the TYs
  if (MTYPE_byte_size(mtype) > MTYPE_byte_size(TY_mtype(fld_ty_idx)))
    fld_ty_idx = MTYPE_To_TY(mtype);
  WN_set_ty (wn, (opr == OPR_ISTORE ?
      Make_Pointer_Type (fld_ty_idx, FALSE) :
      fld_ty_idx));
#ifdef KEY // bug 12394
  if (new_opr == OPR_ILDBITS)
    WN_set_load_addr_ty(wn, Make_Pointer_Type(fld_ty_idx));
#endif
}

static void lower_trapuv_alloca (WN *block, WN *tree, LOWER_ACTIONS actions
#ifdef KEY
         , ST_IDX sym_idx, INT64 offset, TY_IDX pty_idx
         )
#endif
{
  WN *size = WN_kid0(tree);
  WN *con;
#ifdef KEY
  TY_IDX ty_idx;
  if (TY_kind(pty_idx) == KIND_POINTER)
    ty_idx = TY_pointed(pty_idx);
  else if (TY_kind(pty_idx) == KIND_SCALAR &&
     TY_size(pty_idx) == Pointer_Size) {
    // No info on the pointed-to type.  Take a wild guess and make it U4.
    // Bug 8282.
    ty_idx = MTYPE_To_TY(MTYPE_U4);
  } else {
    ty_idx = 0;
    Is_True(FALSE, ("lower_trapuv_alloca: pointer type not found"));
  }
  TYPE_ID type = TY_mtype(ty_idx);
  if (DEBUG_Zero_Uv)      // Initialize vars to 0.
    con = WN_Intconst(MTYPE_byte_size(type)==4 ? MTYPE_U4 : MTYPE_U8, 0);
  else if (MTYPE_byte_size(type) == 4)  // Get 32/64-bit NaN from type.
    con = WN_UVConst(MTYPE_U4);
  else if (MTYPE_byte_size(type) == 8)
    con = WN_UVConst(MTYPE_U8);
  else
#endif
  con = WN_UVConst(WN_rtype(size));

  WN *mstore;
  if (WN_operator(size) == OPR_INTCONST && WN_const_val(size) == 0)
    return; // nothing to do

#ifdef KEY
  ST *sym = &St_Table[sym_idx];
  mstore = WN_CreateMstore(0,
    pty_idx,    // bug 6007
    con,
    WN_Ldid(Pointer_type, offset, sym, ST_type(sym)),
    WN_COPY_Tree(size) );
#else
  mstore = WN_CreateMstore(0,
    Make_Pointer_Type(MTYPE_To_TY(WN_rtype(size)),TRUE),
    con,
    WN_LdidPreg(Pointer_type, Stack_Pointer_Preg_Offset),
    WN_COPY_Tree(size) );
#endif
    mstore = lower_store (block, mstore, actions);
  WN_Set_Linenum(mstore, current_srcpos);  // Bug 1268
  WN_INSERT_BlockLast(block, mstore);
}


inline BOOL Should_Call_Divide(TYPE_ID rtype)
{
#ifdef TARG_X8664
  if( Is_Target_32bit() &&
      ( rtype == MTYPE_I8 || rtype == MTYPE_U8 ) ){
    return TRUE;
  }
#endif // TARG_X8664

#if defined(TARG_IA64)
  if (!OPT_Inline_Divide) {
    if (   rtype == MTYPE_I8 || rtype == MTYPE_U8
  || rtype == MTYPE_I4 || rtype == MTYPE_U4
  || rtype == MTYPE_F4 || rtype == MTYPE_F8) return TRUE;
  }
#elif defined(EMULATE_LONGLONG)
  if (rtype == MTYPE_I8 || rtype == MTYPE_U8) return TRUE;
#endif
#ifdef TARG_SL //expand operator to function call
  return TRUE;
#endif 
  return FALSE;
}

static BOOL Is_Fast_Divide(WN *wn)
{
  OPERATOR opr = WN_operator(wn);
  switch (opr) {
  case OPR_DIV:
  case OPR_REM:
  case OPR_MOD:
    {
      if (WN_operator_is(WN_kid1(wn), OPR_INTCONST)) {
  TYPE_ID rtype = OPCODE_rtype(WN_opcode(wn));
  INT64 constval = WN_const_val(WN_kid1(wn));

  return   opr == OPR_DIV
         ? Can_Do_Fast_Divide(rtype, constval)
         : Can_Do_Fast_Remainder(rtype, constval);
      }
    }
    break;
  }

  return FALSE;
}


#ifdef KEY
/* Convert stmt
      a = a / b
   into
      if( b == N1 ){
         a = a / N1;
      } else if( b == ... ){
         ...;
      } else {
         a = a / b;
      }
   via feedback information.
*/
static WN* simp_remdiv( WN* block, WN* tree )
{
  if( Cur_PU_Feedback == NULL )
    return NULL;

  if( MTYPE_is_float( OPCODE_rtype(WN_opcode(tree) ) ) )
    return NULL;

  const FB_Info_Value& info = Cur_PU_Feedback->Query_value( tree );

  if( info.num_values == 0 )
    return NULL;

  if( info.exe_counter < (float)Div_Exe_Counter )
    return NULL;

  /* Sum up the freq of the top <Div_Exe_Candidates> entries. And perform div
     conversion only if the total freq is not less than <Div_Exe_Ratio>.
  */
  int items = 0;
  const float cutoff_ratio = (float)Div_Exe_Ratio / 100;
  const int num_candidates = MIN( info.num_values, Div_Exe_Candidates );
  FB_FREQ freq(0.0);

  for( items = 1; items <= num_candidates; items++ ){
    freq += info.freq[items-1];
    if( freq / info.exe_counter >= cutoff_ratio )
      break;
  }

  if( freq / info.exe_counter < cutoff_ratio )
    return NULL;

  const TYPE_ID type = WN_rtype( WN_kid(tree,1) );
  const PREG_NUM result = Create_Preg( type, ".remdiv_value" );

  for( int entry = 0; entry < items; entry++ ){
    WN* div_tree = NULL;

    WN* if_then = WN_CreateBlock();
    {
      WN* value = WN_CreateIntconst( OPCODE_make_op(OPR_INTCONST, type, MTYPE_V),
             info.value[entry] );
      WN* tmp = WN_Binary( WN_operator(tree), type,
         WN_COPY_Tree( WN_kid(tree,0) ),
         value );
      WN* stid = WN_StidIntoPreg( type, result, MTYPE_To_PREG( type ), tmp );
      WN_Set_Linenum(stid, current_srcpos);
      WN_INSERT_BlockLast( if_then, stid );
    }
    
    WN* if_else = WN_CreateBlock();
    {
      div_tree = WN_Binary( WN_operator(tree), type,
          WN_COPY_Tree( WN_kid(tree,0) ),
          WN_COPY_Tree( WN_kid(tree,1) ) );
      if( entry+1 == items ){
  WN* stid = WN_StidIntoPreg( type, result, MTYPE_To_PREG( type ), div_tree );
  WN_Set_Linenum(stid, current_srcpos);
  WN_INSERT_BlockLast( if_else, stid );
      }
    }
      
    WN* value = WN_CreateIntconst( OPCODE_make_op(OPR_INTCONST, type, MTYPE_V),
           info.value[entry] );
    WN* if_tree = WN_CreateIf( WN_EQ( type, WN_COPY_Tree(WN_kid(tree,1)),
              value ),
             if_then, if_else );

    Cur_PU_Feedback->Annot( if_tree, FB_EDGE_BRANCH_TAKEN, info.freq[entry] );
    Cur_PU_Feedback->Annot( if_tree, FB_EDGE_BRANCH_NOT_TAKEN,
          ( info.exe_counter - info.freq[entry] ) );
    
    WN_Set_Linenum(if_tree, current_srcpos);
    WN_INSERT_BlockLast( block, if_tree );
    
    WN_Delete( WN_kid(tree,0) );
    WN_Delete( WN_kid(tree,1) );
    WN_Delete( tree );

    tree = div_tree;
    block = if_else;
  }
      
  return WN_LdidPreg( type, result );
}
#endif

#if defined(TARG_SL) && defined(EMULATE_FLOAT_POINT)
static WN *lower_float_unary(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  OPERATOR opr  = WN_operator(tree);
  TYPE_ID desc  = WN_desc(tree);
  TYPE_ID rtype = WN_rtype(tree);
  FmtAssert(((rtype == MTYPE_F4) || (rtype == MTYPE_F8)), ("lower_float_unary: Unexpected type"));  

  WN *iwn = NULL;
  WN *wn = NULL;
  WN *kids[1] = {NULL};
  INTRINSIC intrinsic = INTRINSIC_INVALID; 
  BOOL is_f4 = (rtype == MTYPE_F4);
  switch (opr) {
    case OPR_SQRT:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_SQRT : INTRN_FPE_DOUBLE_SQRT);
      break;
    case OPR_ABS:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_ABS : INTRN_FPE_DOUBLE_ABS);
      break;
    case OPR_NEG:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_NEG : INTRN_FPE_DOUBLE_NEG);   
      break;
    default:
      FmtAssert(FALSE, ("lower_float_unary: Unexpect orerators"));
  }

  FmtAssert(intrinsic != INTRINSIC_INVALID, ("lower_float_unary: Invalid intrinsic"));

  kids[0] = WN_CreateParm(rtype, WN_kid0(tree), MTYPE_To_TY(rtype),
      WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);

  iwn = WN_Create_Intrinsic(OPCODE_make_op(OPR_INTRINSIC_OP, rtype, MTYPE_V),
      intrinsic, 1, kids);

  WN_Delete(tree);  
  return lower_expr(block, iwn, actions); //LOWER_INTRINSIC
}

static WN* lower_float_ALU (WN *block, WN *tree, LOWER_ACTIONS actions)
{
  OPERATOR opr  = WN_operator(tree);
  TYPE_ID desc  = WN_desc(tree);
  TYPE_ID rtype = WN_rtype(tree);

  Is_True(OPCODE_is_expression(WN_opcode(tree)),
      ("lower_float_ALU: expected expression node, not %s", OPCODE_name(WN_opcode(tree))));
  Is_True(actions & LOWER_FP_EMULATE, ("lower_float_ALU: actions does not contain LOWER_FP_EMULATE"));
  Is_True((MTYPE_is_float(rtype)), ("lower_float_ALU: Unexpected float type"));

  WN *iwn = NULL;
  WN *wn  = NULL;
  WN *kids[2] = {NULL, NULL};
  INTRINSIC intrinsic;
  BOOL is_f4 = (rtype == MTYPE_F4);
  switch (opr) {
    case OPR_ADD:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_ADD : INTRN_FPE_DOUBLE_ADD);
      break;
    case OPR_SUB:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_SUB : INTRN_FPE_DOUBLE_SUB);
      break;
    case OPR_MPY:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_MUL : INTRN_FPE_DOUBLE_MUL);
      break;
    case OPR_REM:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_REM : INTRN_FPE_DOUBLE_REM);
      break;
    case OPR_DIV:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_DIV : INTRN_FPE_DOUBLE_DIV);
      break;
    default:
      FmtAssert(FALSE, ("unexpected ALU operator in lower_float_compare")); 
  }

  kids[0] = WN_CreateParm(rtype, WN_kid0(tree), MTYPE_To_TY(rtype),
      WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);
  kids[1] = WN_CreateParm(rtype, WN_kid1(tree), MTYPE_To_TY(rtype),
      WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);  
  iwn = WN_Create_Intrinsic(OPCODE_make_op(OPR_INTRINSIC_OP, rtype, MTYPE_V),
      intrinsic, 2, kids);

  WN_Delete(tree);  
  return lower_expr(block, iwn, actions);//LOWER_INTRINSIC
}

static INTRINSIC SL_CVT_Intrinsic[6][6] = 
{   
  /*I4,    U4,     I8,     U8,      F4,     F8 */
  /* I4 */ {INTRINSIC_NONE, INTRINSIC_NONE, INTRINSIC_NONE, INTRINSIC_NONE, INTRN_FPE_INT32_TO_FLOAT, INTRN_FPE_INT32_TO_DOUBLE},
  /* U4 */ {INTRINSIC_NONE, INTRINSIC_NONE, INTRINSIC_NONE, INTRINSIC_NONE, INTRN_FPE_UINT32_TO_FLOAT, INTRN_FPE_UINT32_TO_DOUBLE},
  /* I8 */ {INTRINSIC_NONE, INTRINSIC_NONE, INTRINSIC_NONE, INTRINSIC_NONE, INTRN_LLE_LL_TO_FLOAT, INTRN_LLE_LL_TO_DOUBLE},
  /* U8 */ {INTRINSIC_NONE, INTRINSIC_NONE, INTRINSIC_NONE, INTRINSIC_NONE, INTRN_LLE_ULL_TO_FLOAT, INTRN_LLE_ULL_TO_DOUBLE},
  /* F4 */ {INTRN_FPE_FLOAT_TO_INT32, INTRN_FPE_FLOAT_TO_UINT32, INTRN_LLE_FLOAT_TO_LL, INTRN_LLE_FLOAT_TO_ULL, INTRINSIC_NONE, INTRN_FPE_FLOAT_TO_DOUBLE},
  /* F8 */ {INTRN_FPE_DOUBLE_TO_INT32, INTRN_FPE_DOUBLE_TO_UINT32, INTRN_LLE_DOUBLE_TO_LL, INTRN_LLE_DOUBLE_TO_ULL, INTRN_FPE_DOUBLE_TO_FLOAT, INTRINSIC_NONE},
};

static INT get_intrinsic_index(TYPE_ID type)
{
  switch (type)
  {
    case MTYPE_I4: return 0;
    case MTYPE_U4: return 1;
    case MTYPE_I8: return 2;
    case MTYPE_U8: return 3;
    case MTYPE_F4: return 4;
    case MTYPE_F8: return 5;
    default:
                   FmtAssert(FALSE, ("get_intrinsic_index: Unexpected type"));
  }
}
static WN *lower_SL_cvt(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  OPERATOR opr = WN_operator(tree);
  TYPE_ID  src = WN_desc(tree);
  TYPE_ID  dst = WN_rtype(tree);
  INTRINSIC intrinsic;

  WN *kids[1] = {NULL};
  WN *iwn;

  FmtAssert((src != dst), ("lower_SL_cvt: expect diff type"));

  INT row = get_intrinsic_index(src);
  INT col = get_intrinsic_index(dst);


  intrinsic = SL_CVT_Intrinsic[row][col];

  kids[0] = WN_CreateParm(src, WN_kid0(tree), MTYPE_To_TY(src),
      WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);

  iwn = WN_Create_Intrinsic(OPCODE_make_op(OPR_INTRINSIC_OP, dst, MTYPE_V),
      intrinsic, 1, kids);

  WN_Delete(tree);  
  return lower_expr(block, iwn, actions); //LOWER_INTRINSIC
}

static WN * lower_float_compare(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  OPERATOR opr  = WN_operator(tree);
  TYPE_ID  desc = WN_desc(tree);

  FmtAssert(((desc == MTYPE_F4) || (desc == MTYPE_F8)), ("lower_float_compare: Unexpected type"));

  WN *iwn     = NULL;
  WN *wn      = NULL;
  WN *kids[2] = {NULL, NULL};
  INTRINSIC intrinsic;
  BOOL is_f4 = (desc == MTYPE_F4);

  switch (opr) {
    case OPR_LT:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_LT : INTRN_FPE_DOUBLE_LT);
      break;
    case OPR_LE:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_LE : INTRN_FPE_DOUBLE_LE);
      break;
    case OPR_EQ:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_EQ : INTRN_FPE_DOUBLE_EQ);
      break;
    case OPR_NE:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_NE : INTRN_FPE_DOUBLE_NE);
      break;
    case OPR_GE:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_GE : INTRN_FPE_DOUBLE_GE);
      break;
    case OPR_GT:
      intrinsic = (is_f4 ? INTRN_FPE_FLOAT_GT : INTRN_FPE_DOUBLE_GT);
      break;    
    default:
      FmtAssert(FALSE, ("unexpected compare condition in lower_float_compare"));
  }

  // new block for lowering intrinsic call properly
  WN *new_block = WN_CreateBlock();  

  kids[0]        = WN_CreateParm(desc, WN_kid0(tree), Be_Type_Tbl(desc),
      WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);
  kids[1]        = WN_CreateParm(desc, WN_kid1(tree), Be_Type_Tbl(desc),
      WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);   
  iwn            = WN_Create_Intrinsic(OPCODE_make_op(OPR_INTRINSIC_CALL,
        MTYPE_I4, MTYPE_V), intrinsic, 2, kids);
  WN_copy_linenum(tree, iwn);
  WN_INSERT_BlockLast(new_block, iwn); // insert into new block


  ST       *preg_st     = MTYPE_To_PREG(MTYPE_I4);
  TY_IDX    preg_ty_idx = Be_Type_Tbl(MTYPE_I4);
  PREG_NUM  preg         = Create_Preg(MTYPE_I4, NULL);

  wn = WN_Ldid(MTYPE_I4, -1, Return_Val_Preg, preg_ty_idx);
  wn = WN_Stid(MTYPE_I4, preg, preg_st, preg_ty_idx, wn),
     WN_copy_linenum(tree, wn);
  WN_INSERT_BlockLast(new_block, wn);  // insert into new block, force use return value of intrinsic call


  WN *low_iwn = lower_intrinsic_call(block, iwn, actions | LOWER_INTRINSIC);

  WN_copy_linenum(tree, low_iwn);
  WN_INSERT_BlockLast(block, low_iwn);  // insert into block

  RETURN_INFO return_info = Get_Return_Info(MTYPE_To_TY(MTYPE_I4), Complex_Not_Simulated);
  PREG_NUM    reg_ret     = RETURN_INFO_preg(return_info, 0);

  wn = WN_Ldid(MTYPE_I4, reg_ret, Return_Val_Preg, preg_ty_idx);
  wn = WN_CreateExp2(OPR_NE, Boolean_type, MTYPE_I4, wn, 
      WN_Intconst(MTYPE_I4, 0));  

  WN_Delete(tree);
  WN_Delete(new_block);

  return lower_expr(block, wn, actions); 
}
#endif //  TARG_SL && EMULATE_FLOAT_POINT

/* ====================================================================
 *
 * WN *lower_expr(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on expression <tree>,
 * returning lowered expression tree.
 *
 * ==================================================================== */

static WN *lower_expr(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  BOOL kids_lowered = FALSE;  /* becomes TRUE when kids are lowered */
  BOOL intrinsic_lowered;

  TYPE_ID type = WN_rtype(tree);

  Is_True(OPCODE_is_expression(WN_opcode(tree)),
    ("expected expression node, not %s", OPCODE_name(WN_opcode(tree))));

  if (OPCODE_is_load(WN_opcode(tree)))
    lower_map(tree, actions);

 /*
  *  before children are processed reassociate for madd oportunities
  */
  if (Action(LOWER_MADD)  &&
      Madd_Allowed    &&
     ((MTYPE_id(type) == MTYPE_F4 || MTYPE_id(type) == MTYPE_F8)
#ifdef TARG_NVISA
      // NVISA supports integer madds
      || (Integer_Madd_Allowed &&
       (MTYPE_id(type) == MTYPE_I4 || MTYPE_id(type) == MTYPE_U4
        || MTYPE_id(type) == MTYPE_I8 || MTYPE_id(type) == MTYPE_U8))
#endif
      ))
  {
    tree = lower_nary_madd(block, tree, actions);
    tree = lower_madd(block, tree, actions);
  }
  if (Action(LOWER_TREEHEIGHT)  &&
      WN_is_commutative(tree))
  {
    if (MTYPE_is_integral(type) || Roundoff_Level >= ROUNDOFF_ASSOC)
      tree = lower_tree_height(block, tree, actions);
  }

 /* Note: We must split constant offsets after lowering complex exprs
  * and splitting symbol addresses since these may create new offsets
  * that need to be split.
  */
  switch (WN_operator(tree))
  {
#if defined(TARG_SL) && defined(EMULATE_LONGLONG)
  case OPR_LSHR: // no break
  case OPR_ASHR:
  case OPR_SHL:  
    if ((Action(LOWER_TO_CG)) && (type == MTYPE_I8 || type == MTYPE_U8))
    {
      OPERATOR opr = WN_operator(WN_kid1(tree));
      if (opr != OPR_INTCONST) // INTCONST is expanded in CG expand
      {  
        INTRINSIC intrinsic = WN_operator(tree) == OPR_LSHR ? INTRN_LLE_LSHR_LL :
          (WN_operator(tree) == OPR_ASHR ? INTRN_LLE_ASHR_LL : INTRN_LLE_ASHL_LL);
        
        WN * kids[2] = {0};
        WN * tmp     = lower_expr(block, WN_kid0(tree), actions);
        TYPE_ID tid  = WN_rtype(tmp);
      
        kids[0]      = WN_CreateParm(tid, tmp, Be_Type_Tbl(tid),
                WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);

        WN * tmp1    = lower_expr(block, WN_kid1(tree), actions);
        TYPE_ID tid1 = WN_rtype(tmp1);
        
        if (tid1 == MTYPE_I8 || tid == MTYPE_U8) 
        {
          tmp1       = WN_Cvt(tid, MTYPE_I4, tmp1);
          tid1       = MTYPE_I4;
        } 
        TY_IDX tyi1  = Be_Type_Tbl(tid1);      
        kids[1]      = WN_CreateParm(tid1, tmp1, tyi1, WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);
      
        WN * iwn     = WN_Create_Intrinsic(
                         OPCODE_make_op(OPR_INTRINSIC_OP, type, MTYPE_V), 
                         intrinsic, 2, kids);
  
        WN_Delete(tree);
        return lower_expr(block, iwn, actions);    
      }
    }
    break;
#endif
  case OPR_INTRINSIC_OP:
    if (INTRN_is_nary(WN_intrinsic(tree)))
      break;

    if (INTRN_cg_intrinsic(WN_intrinsic(tree)))
      break;

#ifdef TARG_X8664
    if( ( (INTRINSIC)WN_intrinsic(tree) == INTRN_F8ANINT ) &&
  Is_Target_SSE2() ){
      /* The code implemented in em_nearest_aint will introduce conversions
   between long long and float, which will slow down the performance
   under -m32.
      */
      if( Is_Target_32bit() )
  break;

      /* cg can generate a more accurate and faster version when
   -OPT:fast_dnint is ON.
   Expand the fast version as late as possible to avoid being
   screwed up by cfold reassociation.
      */
      if( Fast_ANINT_Allowed )
        break;
    }
#endif

#ifdef KEY
    if ( (INTRINSIC) WN_intrinsic (tree) == INTRN_CONSTANT_P
         /* && Action (LOWER_TO_CG) */ ) {
      WN * old = tree;
      WN * parm = WN_kid0 (WN_kid0 (old)); // child of OPR_PARM

      // Check if the argument is a compile-time constant, replace the
      // intrn op by true/false
      if (WN_operator (parm) == OPR_INTCONST ||
          (OPCODE_has_sym (WN_opcode (parm)) &&
           ST_class (WN_st (parm)) == CLASS_CONST)) {
        tree = WN_Intconst (MTYPE_I4, 1);
        WN_copy_linenum (old, tree);
        WN_Delete (old);
        kids_lowered = TRUE;
        break;
      }
      else if ( Action (LOWER_TO_CG) ) {
  // Still not constant, replace it by false
  tree = WN_Intconst (MTYPE_I4, 0);
        WN_copy_linenum (old, tree);
  WN_Delete (old);
  kids_lowered = TRUE;
        break;
      }
    }
#endif

    if (INTRN_is_actual(WN_intrinsic(tree)))
    {
      if (Action(LOWER_INTRINSIC))
      {
  tree = lower_intrinsic(block, tree, actions);
        kids_lowered = TRUE;
      }
      break;
    }
    if (Action(LOWER_INTRINSIC) ||
  Action(LOWER_INLINE_INTRINSIC) ||
#ifdef KEY // bug 6938
  Action(LOWER_FAST_EXP) ||
#endif
  Action(LOWER_INL_STACK_INTRINSIC))
    {
      tree = lower_emulation(block, tree, actions, intrinsic_lowered);
      kids_lowered = TRUE;
    }
    break;

  case OPR_ARRAY:
    if (Action(LOWER_ARRAY))
    {
      tree = lower_linearize_array_addr(block, tree, actions);
      kids_lowered = TRUE;
    }
    break;

  case OPR_ADD:
    if (Action(LOWER_SPLIT_CONST_OFFSETS))
    {
     /*
      * Split
      *       LDA   (c1) <sym>
      *       CONST (c2)
      *     ADD
      * Into
      *       LDA   (hi) <sym>
      *       CONST (low)
      *     ADD
      */
      WN *lda = WN_kid0(tree);
      WN *con = WN_kid1(tree);

      if (WN_operator_is(con, OPR_INTCONST) &&
    foldLdaOffset(lda, WN_const_val(con)))
      {
        WN_OFFSET  offset = WN_lda_offset(lda) + WN_const_val(con);

  if (mem_offset_must_be_split(offset))
  {
    WN_lda_offset(lda) = mem_offset_hi(offset);
    WN_const_val(con) =  mem_offset_lo(offset);
  }
      }
    }
    else
    {
     /*
      * Fold
      *       LDA   (c1) <sym>
      *       CONST (c2)
      *     ADD
      * Into
      *     LDA (c1+c2) <sym>
      */
      WN *lda = WN_kid0(tree);
      WN *con = WN_kid1(tree);

      if (WN_operator_is(con, OPR_INTCONST) &&
    foldLdaOffset(lda, WN_const_val(con)))
      {
  WN_lda_offset(lda) += WN_const_val(con);
  WN_Delete(tree);
  WN_Delete(con);
  tree = lower_expr(block, lda, actions);;
      }
    }
    break;

  case OPR_MLOAD:
    if (Align_Object)
    {
      WN_kid0(tree)=  lower_expr(block, WN_kid0(tree), actions);
      kids_lowered =  TRUE;

      tree = improve_Malignment(tree, WN_kid0(tree), WN_kid1(tree),
        WN_load_offset(tree));
      break;
    }  /* fall thru */

  case OPR_ILOAD:
    if (Action(LOWER_MLDID_MSTID) && WN_opcode(tree) == OPC_MMILOAD)
      return lower_miload(block, tree, actions);

    if (Action(LOWER_BIT_FIELD_ID) && WN_desc(tree) == MTYPE_BS) {
      lower_bit_field_id(tree);
      if (Action(LOWER_BITS_OP) && WN_operator(tree) == OPR_ILDBITS)
  return lower_load_bits (block, tree, actions);
    }

    if (Action(LOWER_SPLIT_CONST_OFFSETS))
    {
     /*
      * Convert
      *      EXPR
      *    ILOAD (offset>16bits)
      * Into
      *        CONST (hi)
      *        EXPR
      *      ADD
      *    ILOAD (lo)
      */
      WN_OFFSET  offset = WN_load_offset(tree);
      if (mem_offset_must_be_split(offset))
      {
  WN_kid0(tree) =  WN_Add(Pointer_type,
        WN_kid0(tree),
        WN_Intconst(Pointer_type,
              mem_offset_hi(offset)));
  WN_load_offset(tree) = mem_offset_lo(offset);
      }
    }
    else
    {
     /*
      * Fold
      *       LDA (c1) <sym>
      *     ILOAD (c2)
      * Into
      *       LDA (0)  <sym>
      *     ILOAD (c1+c2)
      */
      WN *kid = WN_kid0(tree);

#ifdef TARG_MIPS
      /* We don't want this for MIPS since (c1+c2) could likely be
         out-of-range for the ILOAD, and we have the global
         canonicalization optimization to assign c1 and c2
         appropriately. */
      if (FALSE)
#else
      if (foldLdaOffset(kid, WN_load_offset(tree)))
#endif
      {
  WN_load_offset(tree) += WN_lda_offset(kid);
  WN_lda_offset(kid) = 0;
      }
#ifndef TARG_MIPS
     /*
      * Fold
      *           EXPR 
      *           CONST (c1)
      *       ADD expr
      *     ILOAD (c1)
      * Into
      *       EXPR
      *     ILOAD (c1+c2)
      */
      else if (WN_operator_is(kid, OPR_ADD) &&
         foldConstOffset(WN_kid1(kid), WN_load_offset(tree)))
      {
  WN_load_offset(tree) += WN_const_val(WN_kid1(kid));
  WN_kid0(tree) = WN_kid0(kid);
  WN_Delete(WN_kid1(kid));
  WN_Delete(kid);
      }
#endif
    }
    break;

  case OPR_LDID:
    if (Action(LOWER_RETURN_VAL) && WN_st(tree) == Return_Val_Preg)
      return lower_return_ldid(block, tree, actions);

    if (Action(LOWER_MLDID_MSTID) && WN_opcode(tree) == OPC_MMLDID)
      return lower_mldid(block, tree, actions);

    if (Action(LOWER_BIT_FIELD_ID) && WN_desc(tree) == MTYPE_BS) {
      lower_bit_field_id(tree);
      if (Action(LOWER_BITS_OP) && WN_operator(tree) == OPR_LDBITS)
  return lower_load_bits (block, tree, actions);
    }

    if ((WN_class(tree) == CLASS_CONST) && (WN_load_offset(tree) == 0))
    {
      TCON  val = WN_val(tree);
      TYPE_ID valType = WN_val_type(tree);
      WN  *con;

      if (MTYPE_is_integral(type) && MTYPE_is_integral(valType))
      {
  con = WN_Intconst(type, Targ_To_Host( val));
        WN_Delete(tree);
        return con;
      }
     /*
      *  check for real (complex constant) conversion not handled by Targ_Conv
      */
      else if ((MTYPE_is_float(type) && MTYPE_is_float(valType))  &&
        !(!MTYPE_is_complex(type) && MTYPE_is_complex(valType)))
      {
  if (type != valType)
  {
    val = Targ_Conv(type, val);
  }
  con = Make_Const(val);
  WN_Delete(tree);
  return con;
      }
    }
    {
      PREG_NUM last_preg = Get_Preg_Num(PREG_Table_Size(CURRENT_SYMTAB));
      if ((WN_class(tree) == CLASS_PREG) &&
    (WN_load_offset(tree) > last_preg))
      {
    DevWarn("lower_expr() pregno %d > SYMTAB_last_preg(%d)",
      WN_load_offset(tree), last_preg);
      }
    }
   /*
    * Exposes the LDA for RVI usage
    */
    if (Action(LOWER_SPLIT_SYM_ADDRS))
    {
      WN  *iload;
      iload = lower_split_sym_addrs(tree, WN_load_offset(tree), actions);
      if (iload)
      {
  return lower_expr(block, iload, actions);
      }
    }
    if ( Action(LOWER_FORMAL_REF) && WN_class(tree) == CLASS_VAR)
    {
      WN     *iload;

      iload = lower_formal_ref(tree, WN_load_offset(tree),
             WN_st(tree), actions);
      if (iload)
      {
        return lower_expr(block, iload, actions);
      }
    }
    if ( Action(LOWER_UPLEVEL))
    {
      ST *sym = WN_st(tree);

      if (ST_is_uplevelTemp(sym))
      {
        WN     *iload;

        iload = lower_uplevel_reference(tree, WN_load_offset(tree), actions);
  tree = lower_expr(block, iload, actions);
  return tree;
      }
    }
    if ( Action(LOWER_TO_CG) && promote_tls_ldst &&
         ST_is_tls( WN_st(tree) ) ) {
      // There is a ldid to TLS data needs to be promoted to up level
      // In order to avoid conflicts in output registers
      // We create a preg to be the local copy of the TLS variable.
      //       LDID tls_st <ST_IS_THREAD_PRIVATE>
      //     ......
      //   OPR_PARAM
      // OPR_CALL
      // --->
      //   LDID tls_st <ST_IS_THREAD_PRIVATE>
      // STID local_num <PREG>
      //       LDID local_num <PREG>
      //     ......
      //   OPR_PARAM
      // OPR_CALL
      char local_name[64];
      ST* tls_st = WN_st(tree);
      TYPE_ID tls_mtype = ST_mtype(tls_st);
      snprintf(local_name, 64, "%s.local", ST_name(tls_st));
      ST* local_st = MTYPE_To_PREG(tls_mtype);
      PREG_NUM local_num = Create_Preg(tls_mtype, local_name);
      WN* tls_ldid = WN_COPY_Tree(tree);
      WN* local_stid = WN_StidIntoPreg(tls_mtype, local_num,
                                       local_st, tls_ldid);
      WN_INSERT_BlockLast(block, local_stid);
      WN* local_ldid = WN_LdidPreg(tls_mtype, local_num );
      WN_Delete(tree);
      return local_ldid;
    }
    break;

  case OPR_ILDBITS:
  case OPR_LDBITS:
    if (Action(LOWER_BITS_OP))
      return lower_load_bits (block, tree, actions);
    break;
    
  case OPR_LDA:
   /*
    *  use of LDA should mark STFL_ADDR_USED_LOCALLY
    */
    {
      ST *sym = WN_st(tree);

      // if ((ST_class(sym) == CLASS_VAR) ||
      //     (ST_class(sym) == CLASS_FUNC)) {
      //   Do nothing here. ADDR flags should only grow more
      //   optimistic; they should never become more conservative,
      //   because the program's semantics cannot grow worse as we
      //   compile it.
      // }

      if (ST_class(sym) == CLASS_BLOCK && STB_merge(sym))
      {
  DevWarn("LDA (%s) potential bad exposed use of a mergeable symbol",
    ST_name(sym));
      }
    }
    if (Action(LOWER_SPLIT_SYM_ADDRS))
    {
      WN  *lda;
      lda = lower_split_sym_addrs(tree, WN_lda_offset(tree), actions);
      if (lda)
      {
  return lower_expr(block, lda, actions);
      }
    }
    if ( Action(LOWER_FORMAL_REF) && WN_class(tree) == CLASS_VAR)
    {
      WN     *ldid;

      ldid =  lower_formal_ref(tree, WN_lda_offset(tree), WN_st(tree),
         actions);
      if (ldid)
      {
  return lower_expr(block, ldid, actions);
      }
    }
    if ( Action(LOWER_UPLEVEL))
    {
      ST *sym = WN_st(tree);

      if (ST_is_uplevelTemp(sym))
      {
        WN     *ldid;

        ldid = lower_uplevel_reference(tree, WN_lda_offset(tree), actions);
  tree = lower_expr(block, ldid, actions);
  return tree;
      }
    }
    if (Action(LOWER_TO_CG) && promote_tls_ldst &&
        ST_is_tls( WN_st(tree) ) ) 
    {
      // promote OPR_LDA to TLS variable in the OPR_PARAM of OPR_CALL to upper level
      // In order to avoid conflicts in output registers
      //       OPR_LDA tls_st <ST_IS_THREAD_PRIVATE>
      //     ......
      //   OPR_PARAM
      // OPR_CALL
      // --->
      //   OPR_LDA tls_st <ST_IS_THREAD_PRIVATE>
      // OPR_STID local_num <PREG>
      //       OPR_LDID local_num <PREG>
      //     ......
      //   OPR_PARAM
      // OPR_CALL
      char local_name[64];
      ST* tls_st = WN_st(tree);
      snprintf(local_name, 64, "local.%s", ST_name(tls_st));
      ST* local_st = MTYPE_To_PREG(Pointer_type);
      PREG_NUM local_num = Create_Preg ( Pointer_type, local_name);
      WN* tls_lda = WN_COPY_Tree(tree);
      WN* local_stid = WN_StidIntoPreg( Pointer_type, local_num, 
                                        local_st, tls_lda);
      WN_INSERT_BlockLast(block, local_stid);
      WN * local_ldid = WN_LdidPreg(Pointer_type, local_num);
      WN_Delete(tree);
      return local_ldid;
    }
#ifdef KEY  // if taking address of a nested function, replace it by the address
    // of the corresopnding trampoline
    if (Action(LOWER_UPLEVEL) && WN_class(tree) == CLASS_FUNC) {
      ST *sym = WN_st(tree);
      PU &pu = Pu_Table[ST_pu(sym)];
      if (PU_need_trampoline(pu)) {
  Allocate_Object(sym); // because it may not appear again
  ST_IDX tramp_st_idx = find_trampoline(ST_st_idx(sym));
        WN_st_idx(tree) = tramp_st_idx;
  return lower_expr(block, tree, actions | LOWER_UPLEVEL);
      }
    }
#endif

    break;

  case OPR_CVT:
  case OPR_TRUNC:
  case OPR_RND:
    if (Action(LOWER_CVT))
    {
      tree = lower_cvt(block, tree, actions);
      kids_lowered = TRUE;
    }
    break;

  case OPR_TAS:

    /* If the operand of the TAS is a load from memory, try to
     * replace with a load matching the result type of the TAS.
     * Doing so may avoid a move from one register set to another.
     */
    if (Action(LOWER_TO_CG))
    {
      WN *load = WN_kid0(tree);
      if (   OPERATOR_is_load(WN_operator(load))
    && (!WN_has_sym(load) || WN_class(load) != CLASS_PREG))
      {
  TYPE_ID tas_rtype = WN_rtype(tree);
  if (MTYPE_byte_size(tas_rtype) == MTYPE_byte_size(WN_desc(load))) {
    WN_set_rtype(load, tas_rtype);
    WN_set_desc(load, tas_rtype);
    WN_Delete(tree);
    return lower_expr(block, load, actions);
  }
      }
    }
    break;

  case OPR_IMAGPART:
    if (Action(LOWER_COMPLEX))
    {
      WN  *realexp, *imagexp;
      lower_complex_expr(block, WN_kid0(tree), actions, &realexp, &imagexp);
      WN_Delete(tree);
      tree = lower_expr(block, imagexp, actions);
    }
    break;

  case OPR_REALPART:
    if (Action(LOWER_COMPLEX))
    {
      WN  *realexp, *imagexp;

      lower_complex_expr(block, WN_kid0(tree), actions, &realexp, &imagexp);
      WN_Delete(tree);
      tree = lower_expr(block, realexp, actions);
    }
    break;
    
  case OPR_EQ:
    if (Action(LOWER_COMPLEX) && MTYPE_is_complex(WN_desc(tree)))
    {
      /*
       *  x == y
       *    R(x)==R(y) && I(x)==I(y)
       */
      WN  *rx, *ry, *ix, *iy;
      TYPE_ID realTY = Mtype_complex_to_real( WN_desc(tree));

      lower_complex_expr(block, WN_kid0(tree), actions, &rx, &ix);
      lower_complex_expr(block, WN_kid1(tree), actions, &ry, &iy);
 
      tree = WN_LAND(WN_EQ(realTY, rx, ry),
         WN_EQ(realTY, ix, iy));

      return lower_expr(block, tree, actions);
    }
    break;

  case OPR_NE:
    if (Action(LOWER_COMPLEX) && MTYPE_is_complex(WN_desc(tree)))
    {
      /*
       *  x != y
       *    ! ( R(x)==R(y)  &&  I(x)==I(y) )
       */
      WN  *rx, *ry, *ix, *iy;
      TYPE_ID realTY = Mtype_complex_to_real( WN_desc(tree));

      lower_complex_expr(block, WN_kid0(tree), actions, &rx, &ix);
      lower_complex_expr(block, WN_kid1(tree), actions, &ry, &iy);
 
      tree = WN_LNOT(WN_LAND(WN_EQ(realTY, rx, ry),
           WN_EQ(realTY, ix, iy)));

      return lower_expr(block, tree, actions);
    }
    break;

  case OPR_MADD:
    if (Action(LOWER_QUAD) && MTYPE_is_quad(type))
    {
     /*
      *   kid1 * kid2 + kid0
      */
      WN  *wn;

      wn = WN_Add(type,
      WN_Mpy(type, WN_kid1(tree), WN_kid2(tree)),
      WN_kid0(tree));
      WN_Delete(tree);

      tree = wn;
    }
    break;

  case OPR_MSUB:
    if (Action(LOWER_QUAD) && MTYPE_is_quad(type))
    {
     /*
      *   kid1 * kid2 - kid0
      */
      WN  *wn;

      wn = WN_Sub(type,
      WN_Mpy(type, WN_kid1(tree), WN_kid2(tree)),
      WN_kid0(tree));
      WN_Delete(tree);

      tree = wn;
    }
    break;

  case OPR_NMADD:
    if (Action(LOWER_QUAD) && MTYPE_is_quad(type))
    {
     /*
      *   - (kid1 * kid2 + kid0)
      */
      WN  *wn, *madd;

      madd = WN_Add(type,
        WN_Mpy(type, WN_kid1(tree), WN_kid2(tree)),
        WN_kid0(tree));

      wn = WN_Neg(type, madd);
      WN_Delete(tree);

      tree = wn;
    }
    break;

  case OPR_NMSUB:
    if (Action(LOWER_QUAD) && MTYPE_is_quad(type))
    {
     /*
      *   - (kid1 * kid2 - kid0)  -->   (kid0 - kid1 * kid2)
      */
      WN  *wn;

      wn = WN_Sub(type,
      WN_kid0(tree),
      WN_Mpy(type, WN_kid1(tree), WN_kid2(tree)));
      WN_Delete(tree);

      tree = wn;
    }
    break;

  case OPR_RSQRT:
    tree = lower_rsqrt(block, tree, actions);
    break;

  case OPR_RECIP:
    tree = lower_recip(block, tree, actions);
    break;

  case OPR_SELECT:
    {
#ifdef KEY // f90 front-end sometimes generate inconsistent types; fix them
      if (MTYPE_size_min(type) != MTYPE_size_min(WN_rtype(WN_kid1(tree))))
  WN_kid1(tree) = WN_Cvt(WN_rtype(WN_kid1(tree)), type, WN_kid1(tree));
      if (MTYPE_size_min(type) != MTYPE_size_min(WN_rtype(WN_kid2(tree))))
  WN_kid2(tree) = WN_Cvt(WN_rtype(WN_kid2(tree)), type, WN_kid2(tree));
#endif
      WN * const kid0 = WN_kid0(tree);  // the condition expression
      if (WN_operator_is(kid0, OPR_INTCONST))
      {
  INT64 flag = WN_const_val(kid0);
  WN * const kid = flag ? WN_kid1(tree) : WN_kid2(tree);
  return lower_expr(block, kid, actions);
      } else if (WN_operator(kid0) == OPR_LNOT) {
  // swap the select inputs and replace the condition with
  // operand of the LNOT. Then lower the whole tree again
  // as there may be further lowerings that may occur with the
  // new operands.
  WN * const new_kid1 = WN_kid2(tree);
  WN * const new_kid2 = WN_kid1(tree);
  WN * const new_kid0 = WN_kid0(kid0);
  TYPE_ID new_desc = (WN_rtype(new_kid0) == MTYPE_B) ? MTYPE_B : MTYPE_V;
  WN_kid0(tree) = new_kid0;
  WN_kid1(tree) = new_kid1;
  WN_kid2(tree) = new_kid2;
  WN_set_desc(tree, new_desc);
  WN_Delete(kid0);
  return lower_expr(block, tree, actions);
      }
    }
    break;

  case OPR_PAREN:
    if (Roundoff_Level > ROUNDOFF_ASSOC
#ifdef TARG_X8664 // Such useless PARENs are used by LNO's Simd phase as markers
        || WN_kid_count(tree) < 2
#endif
       )
    {
     /*
      *  At suitable roundoff we may remove these parens
      *  This will allow better MADD generation latter (pv 316380)
      */
      WN *kid0 = WN_kid0(tree);

      WN_Delete(tree);
      return lower_expr(block, kid0, actions);
    }
    break;

  case OPR_DIV:
  case OPR_REM:
  case OPR_MOD:

    {
#ifdef KEY
      WN* simp = simp_remdiv( block, tree );
      if( simp != NULL ){
  return simp;
      }
#endif      

      /* If not inlining divides, then generate an INTRINSIC_OP that is
       * later lowered to a call
       */
      TYPE_ID rtype = OPCODE_rtype(WN_opcode(tree));
      if (Should_Call_Divide(rtype) && !Is_Fast_Divide(tree)) {

#ifdef EMULATE_LONGLONG
        if (rtype == MTYPE_I8 || rtype == MTYPE_U8) {
    FmtAssert (OPCODE_rtype(WN_opcode(WN_kid0(tree))) == rtype,
         ("DIV/REM/MOD: kid0 should be %d, is %d",
         rtype, OPCODE_rtype(WN_opcode(WN_kid0(tree)))));
    FmtAssert (OPCODE_rtype(WN_opcode(WN_kid1(tree))) == rtype,
         ("DIV/REM/MOD: kid1 should be %d, is %d",
         rtype, OPCODE_rtype(WN_opcode(WN_kid1(tree)))));
  }
#endif

  WN *kids[2];
  WN *iwn;
  LEAF tmpY;
  INTRINSIC intrinsic;
  BOOL is_unsigned = MTYPE_is_unsigned(rtype);
  BOOL is_float = MTYPE_is_float(rtype);
  BOOL is_double = MTYPE_is_size_double(rtype);
  switch (WN_operator(tree)) {
  case OPR_DIV:
    if (is_float) {
#ifdef TARG_SL
      intrinsic = (is_double ? INTRN_FPE_DOUBLE_DIV : INTRN_FPE_FLOAT_DIV);
#else
      intrinsic = (is_double ? INTRN_DIVDF3 : INTRN_DIVSF3);
#endif
    } else if (is_double) {
      intrinsic = (is_unsigned ? INTRN_UDIVDI3 : INTRN_DIVDI3);
    } else {
      intrinsic = (is_unsigned ? INTRN_UDIVSI3 : INTRN_DIVSI3);
    }
    break;
  case OPR_MOD:
    FmtAssert(!is_float, ("Unexpected MOD operator on float"));
    // Unsigned MOD is the same as REM.
    // Signed MOD is a REM followed by an adjustment which
    // uses the divisor, so save it to a temp and replace the
    // divisor operand of the tree with a load from the temp.
    if (!is_unsigned) {
      tmpY = Make_Leaf(block, WN_kid1(tree), type);
      WN_kid1(tree) = Load_Leaf(tmpY);
    }
    /*FALLTHROUGH*/
  case OPR_REM:
    FmtAssert(!is_float, ("Unexpected REM operator on float"));
    if (is_double) {
      intrinsic = (is_unsigned ? INTRN_UMODDI3 : INTRN_MODDI3);
    } else {
      intrinsic = (is_unsigned ? INTRN_UMODSI3 : INTRN_MODSI3);
    }
    break;
  default:
    #pragma mips_frequency_hint NEVER
    FmtAssert (FALSE, ("Unexpected division operator"));
    /*NOTREACHED*/
  }
  kids[0] = WN_CreateParm (rtype,
         lower_expr(block, WN_kid0(tree), actions),
         Be_Type_Tbl(rtype),
         WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);
  kids[1] = WN_CreateParm (rtype, 
         lower_expr(block, WN_kid1(tree), actions),
         Be_Type_Tbl(rtype),
         WN_PARM_BY_VALUE | WN_PARM_READ_ONLY);
  iwn = WN_Create_Intrinsic(OPCODE_make_op(OPR_INTRINSIC_OP,
             rtype, MTYPE_V),
          intrinsic, 2, kids);

  if (WN_operator(tree) == OPR_MOD && !is_unsigned) {
    // For signed MOD, we need to add the divisor to the result
    // of the REM if both of the operands are negative.
    WN *t2, *t3, *t4;
    PREG_NUM t1;
    t1 = AssignExpr(block, iwn, type);
    t2 = WN_Bxor(type, WN_LdidPreg(type, t1), Load_Leaf(tmpY));
    t3 = WN_Ashr(type, t2, WN_Intconst(type, MTYPE_size_reg(type) - 1));
    t4 = WN_Band(type, Load_Leaf(tmpY), t3);
    iwn = WN_Add(type, WN_LdidPreg(type, t1), t4);
    iwn = lower_expr(block, iwn, actions);
  }

#ifdef TARG_X8664
  if( Action( LOWER_TO_CG ) &&
      Action( LOWER_INTRINSIC) ){
    BOOL intrinsic_lowered = FALSE;
    iwn = lower_emulation( block, iwn, actions, intrinsic_lowered );
  }
#endif

  WN_Delete(tree);
  return iwn;
      }
    }

    if (Action(LOWER_COMPLEX) && MTYPE_is_complex(WN_rtype(tree))) {
  // complex div creates if-then-else structure,
  // so want to expand this early, then just return C*LDID preg here
  // (e.g. in case is under a PARM node, and is not first parm).
  // Note that fortran fe moves this out from under call, 
  // but C doesn't.  Apparently only get here for C PARM case.
  TYPE_ID cdiv_mtype = WN_rtype(tree);
  // using a preg causes wopt to get confused and not
  // connect the F* preg numbers with the C* preg number.
  // so use temp symbol instead.
  TY_IDX cdiv_ty = MTYPE_To_TY(cdiv_mtype);
  ST *cdiv_st = Gen_Temp_Symbol (cdiv_ty, ".complex_div");
  WN *stid = WN_Stid(cdiv_mtype, 0, cdiv_st, cdiv_ty, tree);
        WN_Set_Linenum (stid, current_srcpos);
  stid = lower_store(block, stid, actions);
  WN_INSERT_BlockLast(block, stid);
  WN *ldid = WN_Ldid(cdiv_mtype, 0, cdiv_st, cdiv_ty);
  return ldid;
    }
    break;

  case OPR_COMMA:
    {
      WN *commaBlock;
      commaBlock = lower_block(WN_kid0(tree), actions);

      DevWarn("lower_expr(): comma operator seen, line %d",
        Srcpos_To_Line(current_srcpos));

#ifdef KEY
      // Bug 1268 - copy linenumber when creating WNs.
      WN_copy_linenum(tree, commaBlock);
#endif
      WN_INSERT_BlockLast(block, commaBlock);
    }
    return lower_expr(block, WN_kid1(tree), actions);

  case OPR_CSELECT:
   /*
    *  
    */
    if (Action(LOWER_SHORTCIRCUIT))
    {
      /*******************************************************************

      DevWarn("lower_expr(): cselect operator seen, line %d",
              Srcpos_To_Line(current_srcpos));
      *******************************************************************/

      if (expr_is_speculative(tree))
      {
  WN *select = WN_Select(type, WN_kid0(tree), WN_kid1(tree),
             WN_kid2(tree));
  
  WN_Delete(tree);
  return select;
      }
      else
      {
  PREG_NUM  tmpN;
  WN *if_tree, *if_then, *if_else, *stid;
  WN *body = WN_CreateBlock();

  if_then = WN_CreateBlock();
  tmpN = AssignExpr(if_then, WN_kid1(tree), type);

  if_else = WN_CreateBlock();
  stid = WN_StidIntoPreg(type, tmpN, MTYPE_To_PREG(type), WN_kid2(tree));
  WN_Set_Linenum(stid, current_srcpos);  // Bug 1268
  WN_INSERT_BlockLast(if_else, stid);

  if_tree = WN_CreateIf( WN_kid0(tree), if_then, if_else );
  if ( Cur_PU_Feedback )
    Cur_PU_Feedback->FB_lower_branch( tree, if_tree );
  WN_INSERT_BlockLast( block, lower_if( body, if_tree, actions ) );

  return WN_LdidPreg(type, tmpN);
      }
    }
    break;

  case OPR_CAND:
  case OPR_CIOR:
   /*
    *  return boolean 0/1 (build CSELECT)
    */
    if (Action(LOWER_SHORTCIRCUIT))
    {
      if (expr_is_speculative(tree))
      {
  WN *cond;

  if (WN_operator_is(tree, OPR_CAND))
    cond = WN_LAND( WN_kid0(tree), WN_kid1(tree));
  else
    cond = WN_LIOR( WN_kid0(tree), WN_kid1(tree));

  WN_Delete(tree);
  return lower_expr(block, cond, actions);
      }
      else
      {
  WN *select = WN_Cselect(type,
        tree, 
        WN_Intconst(Boolean_type, 1),
        WN_Intconst(Boolean_type, 0));
  return lower_expr(block, select, actions);
      }
    }
    break;

#ifdef KEY
  case OPR_PURE_CALL_OP:
    if (Action(LOWER_TO_CG))
    {
      // create and insert call
      WN * call = WN_Create (OPR_CALL,
                             WN_rtype (tree),
           WN_desc (tree),
           WN_kid_count (tree));
      WN_st_idx (call) = WN_st_idx (tree);
      for (int i=0; i<WN_kid_count (call); i++)
        WN_kid (call, i) = WN_kid (tree, i);
      WN_Set_Linenum(call, current_srcpos);

      call = lower_call (block, call, actions);
      WN_INSERT_BlockLast (block, call);

      // ldid return val
      WN * ldid = WN_Ldid (WN_rtype (call), -1, Return_Val_Preg, MTYPE_TO_TY_array[WN_rtype (call)]);
      ldid = lower_return_ldid (block, ldid, actions);

      // stid return val into temporary
      ST * sym = Gen_Temp_Symbol (WN_ty (ldid), "pure_call_ret");
      WN * stid = WN_Stid (WN_rtype (ldid), 0, sym, WN_ty (ldid), ldid);
      WN_Set_Linenum(stid, current_srcpos);
      stid = lower_store (block, stid, actions);
      WN_INSERT_BlockLast (block, stid);

      // ldid val from temporary
      ldid = WN_Ldid (WN_desc (stid), 0, sym, WN_ty (ldid));
      WN_Delete (tree);

      tree = lower_expr (block, ldid, actions);

    }
    break;
#endif
  }

#if defined(TARG_SL) && defined(EMULATE_FLOAT_POINT)
  if (Action(LOWER_FP_EMULATE)) {

    TYPE_ID desc = WN_desc(tree);

    switch (WN_operator(tree)) {
      case OPR_ADD:
      case OPR_SUB:
      case OPR_MPY:
      case OPR_REM:
      case OPR_DIV: 
        if (MTYPE_is_float(type)) {
          return lower_float_ALU(block, tree, actions);
        }
        break;

      case OPR_NE:
      case OPR_EQ:
      case OPR_GE:
      case OPR_GT:
      case OPR_LE:
      case OPR_LT:  
        if (MTYPE_is_float(desc)) {
          return lower_float_compare(block, tree, actions);
        }
        break;

     case OPR_NEG:
      case OPR_ABS:
        /* Expanded in lowering to CG phase */
        break;
      
      case OPR_SQRT:
        if (MTYPE_is_float(type)) {
          return lower_float_unary(block, tree, actions);
        }
        break;  

      case OPR_FLOOR:
      case OPR_CEIL:
      case OPR_TRUNC:
      case OPR_RND:
      case OPR_CVT:
        if ((MTYPE_is_float(type) || MTYPE_is_float(desc))
            && (desc != type)) {
          return lower_SL_cvt(block, tree, actions);
        }
    }
  }
#endif

  if (Action(LOWER_QUAD))
  {
    if (WN_desc(tree) == MTYPE_FQ)
    {
      switch (WN_operator(tree))
      {
      case OPR_CONST:
      case OPR_LDID:
      case OPR_ILOAD:
  break;
      case OPR_EQ:
      case OPR_NE:
      case OPR_LE:
      case OPR_LT:
      case OPR_GT:
      case OPR_GE:
      case OPR_CVT:
      case OPR_TRUNC:
      case OPR_RND:
      case OPR_CEIL:
      case OPR_FLOOR:
  tree = lower_emulation(block, tree, actions,intrinsic_lowered);
  break;
      default:
  break;
      }
    }
    if (WN_rtype(tree) == MTYPE_FQ)
    {
      switch (WN_operator(tree))
      {
      case OPR_CONST:
      case OPR_LDID:
      case OPR_ILOAD:
  break;

      case OPR_SELECT:
      case OPR_NEG:
      case OPR_ABS:
      case OPR_SQRT:
      case OPR_ADD:
      case OPR_SUB:
      case OPR_MPY:
      case OPR_DIV:
      case OPR_MOD:
      case OPR_REM:
      case OPR_MAX:
      case OPR_MIN:
      case OPR_CVT:
      case OPR_TRUNC:
      case OPR_RND:
  tree = lower_emulation(block, tree, actions, intrinsic_lowered);
  break;
      default:
  break;
      }
    }
  }

  if (WN_nary_intrinsic(tree))
  {
    tree = WN_NaryToExpr(tree);
  }

  /* Lower kids if not done already. */
  if (! kids_lowered)
  {
     INT16 i;
     for (i = 0; i < WN_kid_count(tree); i++)
       WN_kid(tree,i) = lower_expr(block, WN_kid(tree,i), actions);
     tree = WN_Simplify_Rebuild_Expr_Tree(tree,alias_manager);
  }

  return tree;
}


/* ====================================================================
 *
 *  static TY_IDX coerceTY(TY_IDX type, TYPE_ID btype)
 *
 * return TY corresponding to the btype
 * (type might be pointer -> btype)
 *
 * ==================================================================== */

static TY_IDX coerceTY(TY_IDX type, TYPE_ID btype)
{
  TY &ty = Ty_Table[type];

  if (TY_is_pointer(ty))
    return Make_Pointer_Type(coerceTY(TY_pointed(ty), btype));

  return MTYPE_To_TY(btype);
}




/* ====================================================================
 *
 *  static ST * coerceST(const ST *st, TYPE_ID type)
 *
 * return ST corresponding to the type
 *
 * ==================================================================== */

static ST *coerceST(const ST *st, TYPE_ID type)
{
  if (ST_class(st) == CLASS_PREG)
  {
   /*
    *  for now, only pregs must correspond to the type
    */
    return MTYPE_To_PREG(type);
  }

  return (ST *) st;
}

static ST *coerceST(const ST &st, TYPE_ID type)
{
  if (ST_class(&st) == CLASS_PREG)
  {
   /*
    *  for now, only pregs must correspond to the type
    */
    return MTYPE_To_PREG(type);
  }

  return (ST *) &st;
}


/* ====================================================================
 *
 * static BOOL WN_StoreIsUnused(WN *tree)
 *
 * Find if store has been marked by IPA as unused
 * This may require traversal of the address expression to find
 * an LDA or ARRAY
 * ==================================================================== */
static BOOL WN_StoreIsUnused(WN *tree)
{
  ST  *sym;

  switch(WN_operator(tree))
  {
  case OPR_LDA:
  case OPR_STID:
    sym = WN_st(tree);

    if (ST_class(sym) != CLASS_PREG  &&
        ST_class(sym) != CLASS_BLOCK &&
  ST_is_not_used(sym))
      return TRUE;
    break;

  case OPR_ARRAY:
    return WN_StoreIsUnused(WN_array_base(tree));
  }

  return FALSE;
}

/* ====================================================================
 *
 * WN *add_fake_parm(WN *o_call, WN *fake_actual)
 *
 * Add the fake actual parameter as the first parameter to the original call.
 * All original parameters are shifted down by 1.
 *
 * ==================================================================== */
static WN *add_fake_parm(WN *o_call, WN *fake_actual, TY_IDX ty_idx)
{
  WN *n_call;
  if (WN_operator(o_call) == OPR_ICALL)
    n_call = WN_Icall(MTYPE_V, MTYPE_V, WN_kid_count(o_call)+1, WN_ty(o_call));
  else
    n_call = WN_generic_call(WN_operator(o_call), MTYPE_V, MTYPE_V, 
           WN_kid_count(o_call)+1, WN_st_idx(o_call));
  WN_call_flag(n_call) = WN_call_flag(o_call);
  WN_Set_Linenum(n_call, WN_Get_Linenum(o_call));
  if ( Cur_PU_Feedback ) {
    Cur_PU_Feedback->FB_lower_call( o_call, n_call );
  }
  WN_kid0(n_call) = WN_CreateParm(Pointer_Mtype, fake_actual, ty_idx,
            WN_PARM_BY_REFERENCE | WN_PARM_PASSED_NOT_SAVED);
  for (INT32 i = 0; i < WN_kid_count(o_call); i++)
    WN_kid(n_call, i+1) = WN_COPY_Tree(WN_kid(o_call, i));
  return n_call;
}

/* ====================================================================
 *
 * WN *lower_return_mstid(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * "tree" must be an MSTID whose kid is MLDID of Return_Val_Preg (-1).
 * Perform lowering of MSTID whose rhs is Return_Val_Preg; translate to
 * either MSTORE or a sequence of STIDs.
 *
 * ==================================================================== */
static WN *lower_return_mstid(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  ST *preg_st;
  WN *n_rhs;
  WN *wn = NULL;  // init to prevent upward-exposed use
  RETURN_INFO return_info = Get_Return_Info(WN_ty(tree), Complex_Not_Simulated
#ifdef TARG_X8664
              , last_call_ff2c_abi
#endif
               );
  if (RETURN_INFO_return_via_first_arg(return_info)) { // fake first parm
    // get the previous MCALL statement
    WN *call = WN_last(block);
    Is_True(WN_operator(call) == OPR_CALL || WN_operator(call) == OPR_ICALL ||
      WN_operator(call) == OPR_PICCALL,
      ("statement preceding MMLDID of Return_Val_Preg must be a call"));
#ifndef TARG_X8664
    Is_True(WN_rtype(call) == MTYPE_M,
      ("call preceding MMLDID of Return_Val_Preg not type M"));
#else
    Is_True(WN_rtype(call) == MTYPE_M || MTYPE_is_complex(WN_rtype(call)) ||
          MTYPE_is_mmx_vector(WN_rtype(call)),
      ("call preceding MMLDID of Return_Val_Preg not type M or complex"));
#endif
    WN *awn = WN_CreateLda(OPR_LDA, Pointer_Mtype, MTYPE_V, 
         WN_store_offset(tree), 
         Make_Pointer_Type(WN_ty(tree)), WN_st_idx(tree));
    awn = lower_expr(block, awn, actions);
    WN *n_call = add_fake_parm(call, awn, WN_ty(awn));
    WN_DELETE_FromBlock(block, call);
    WN_INSERT_BlockLast(block, n_call); 

    WN_DELETE_Tree(tree);
    return NULL; // original MSTID disappears
  }
  else { // return via 1 or more return registers
    TYPE_ID mtype, desc;
    for (INT32 i = 0; i < RETURN_INFO_count(return_info); i++) {
      if (i != 0)
        WN_INSERT_BlockLast (block, wn); // insert the last STID created 
      mtype = RETURN_INFO_mtype(return_info, i);
      desc = mtype;
#ifdef KEY
      if (MTYPE_byte_size(mtype) > TY_size(WN_ty(tree)) && !MTYPE_float(mtype)){
  desc = Mtype_AlignmentClass(1, MTYPE_type_class(mtype));
  while (MTYPE_byte_size(desc) < TY_size(WN_ty(tree)))
    desc = Mtype_next_alignment(desc);
      }
#endif
      preg_st = Standard_Preg_For_Mtype(mtype);
      n_rhs = WN_CreateLdid(OPR_LDID, mtype, mtype, 
          RETURN_INFO_preg(return_info, i), preg_st,
          Be_Type_Tbl(mtype));
#ifndef KEY
      if (TY_align(ST_type(WN_st(tree))) < MTYPE_alignment(mtype)) {
  DevWarn("return_info struct alignment is smaller than register size, may produce wrong results");
      }
#endif
      wn = WN_CreateStid(OPR_STID, MTYPE_V, desc, 
             WN_store_offset(tree)+i*MTYPE_byte_size(mtype),
       WN_st_idx(tree), Be_Type_Tbl(desc), n_rhs);
      wn  = lower_store (block, wn, actions);
      WN_Set_Linenum(wn, current_srcpos);
    }
    WN_DELETE_Tree(tree);
    return wn;
  }
}

/* ====================================================================
 *
 * WN *lower_return_mistore(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * "tree" must be an MISTORE whose rhs is MLDID of Return_Val_Preg (-1).
 * Perform lowering of MISTORE whose rhs is Return_Val_Preg; translate to
 * either MSTORE or a sequence of ISTOREs.
 *
 * ==================================================================== */
static WN *lower_return_mistore(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  TYPE_ID mtype;
  ST *preg_st;
  WN *n_rhs;
  WN *wn = NULL;  // init to prevent upward-exposed use
  RETURN_INFO return_info = Get_Return_Info(WN_ty(tree), Complex_Not_Simulated
#ifdef TARG_X8664
              , last_call_ff2c_abi
#endif
             );
  if (RETURN_INFO_return_via_first_arg(return_info)) { // fake first parm
    // get the previous MCALL statement
    WN *call = WN_last(block);
    Is_True(WN_operator(call) == OPR_CALL || WN_operator(call) == OPR_ICALL ||
      WN_operator(call) == OPR_PICCALL,
      ("statement preceding MMLDID of Return_Val_Preg must be a call"));
#ifndef TARG_X8664
    Is_True(WN_rtype(call) == MTYPE_M,
      ("call preceding MMLDID of Return_Val_Preg not type M"));
#else
    Is_True(WN_rtype(call) == MTYPE_M || MTYPE_is_mmx_vector(WN_rtype(call)),
      ("call preceding MMLDID of Return_Val_Preg not type M"));
#endif
    WN *awn = WN_COPY_Tree(WN_kid1(tree));
    if (WN_store_offset(tree) != 0) { // generate an ADD node for the offset
      WN *iwn = WN_CreateIntconst(OPR_INTCONST, Pointer_Mtype, MTYPE_V, 
          WN_store_offset(tree));
      awn = WN_CreateExp2(OPR_ADD, Pointer_Mtype, Pointer_Mtype, awn, iwn);
    }
    awn = lower_expr(block, awn, actions);
    WN *n_call = add_fake_parm(call, awn, WN_ty(tree));
    WN_DELETE_FromBlock(block, call);
    WN_INSERT_BlockLast(block, n_call); 

    WN_DELETE_Tree(tree);
    return NULL; // original MSTID disappears
  }
  else { // return via 1 or more return registers
    WN *base_expr;
    for (INT32 i = 0; i < RETURN_INFO_count(return_info); i++) {
      if (i != 0)
        WN_INSERT_BlockLast (block, wn); // insert the last STID created 
      mtype = RETURN_INFO_mtype(return_info, i);
      preg_st = Standard_Preg_For_Mtype(mtype);
      n_rhs = WN_CreateLdid(OPR_LDID, mtype, mtype, 
          RETURN_INFO_preg(return_info, i), preg_st,
          Be_Type_Tbl(mtype));
      base_expr = WN_COPY_Tree(WN_kid1(tree));
// OSP-438: The type of Istore node should be a pointer to mtype.
// (Previously, we just use Be_Type_Tbl(mtype) as the type of Istore).
      wn = WN_CreateIstore(OPR_ISTORE, MTYPE_V, mtype, 
               WN_store_offset(tree)+i*MTYPE_byte_size(mtype),
         Make_Pointer_Type(Be_Type_Tbl(mtype), FALSE), n_rhs, base_expr);
      WN_Set_Linenum(wn, WN_Get_Linenum(tree));
      wn  = lower_store (block, wn, actions);
      WN_Set_Linenum (wn, WN_Get_Linenum(tree));
    }
    WN_DELETE_Tree(tree);
    return wn;
  }
}

#ifdef TARG_NVISA
// lower struct copy of asm output, which goes downward from negative preg,
// into sequence of ldid/stid. 
static WN *lower_asm_mstid(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  WN *wn;
  WN *ldid;
  INT field_id = 0;
  TY_IDX vty = WN_ty(tree);
  FmtAssert(TY_can_be_vector(vty), ("not a vector type"));
  TY_IDX ety = TY_vector_elem_ty(vty);
  PREG_NUM pnum = WN_load_offset(WN_kid0(tree)); // asm out preg
  for (INT i = 0; i < TY_vector_count(vty); ++i) {
      ldid = WN_CreateLdid(OPR_LDID, TY_mtype(ety), TY_mtype(ety), 
    /* asm out pregs count downward */
    pnum - i, 
    MTYPE_To_PREG(TY_mtype(ety)), ety);
      ++field_id;
      wn = WN_CreateStid( OPR_STID, MTYPE_V, TY_mtype(ety),
    /* increment by size of element */
    i * MTYPE_byte_size(TY_mtype(ety)), 
    WN_st(tree), vty, ldid, field_id);
      wn  = lower_store (block, wn, actions);
      WN_Set_Linenum (wn, WN_Get_Linenum(tree));
      WN_INSERT_BlockLast (block, wn); // insert the last STID created 
  }
  WN_DELETE_Tree(tree);
  return NULL; // original MSTID disappears
}
#endif // TARG_NVISA

/* ====================================================================
 *
 * WN *lower_mstid(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on MSTID nodes returning
 * an equivalent MSTORE node.
 *
 * ==================================================================== */

static WN *lower_mstid(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  TY_IDX ty_idx  = WN_ty(tree);
  TY_IDX pty_idx;
  UINT64 size    = WN_field_id(tree) == 0 ?
      TY_size(ty_idx):
      TY_size(get_field_type (ty_idx, WN_field_id(tree)));
  WN*    wn;
  WN*    awn;
  WN*    swn;

  Is_True((WN_operator(tree) == OPR_STID && WN_desc(tree) == MTYPE_M),
    ("expected mstid node, not %s", OPCODE_name(WN_opcode(tree))));

  pty_idx = Make_Pointer_Type (ty_idx, FALSE);

  WN *rhs = WN_kid0(tree);
  if (WN_opcode(rhs) == OPC_MMLDID && WN_st(rhs) == Return_Val_Preg) {
    // handle lowering of MLDID of Return_Val_Preg followed by MSTID
    Is_True(Action(LOWER_RETURN_VAL),
      ("LOWER_RETURN_VAL action must be specified"));
    return lower_return_mstid(block, tree, actions);
  }

#ifdef KEY
  // Don't do struct copy if src and dest are the same.
  if (WN_opcode(rhs) == OPC_MMLDID &&
      WN_st(rhs) == WN_st(tree) &&
      WN_load_offset(rhs) == WN_store_offset(tree)) {
    WN_Delete(tree);
    return NULL;
  }
#endif

  awn = WN_CreateLda (OPR_LDA, Pointer_Mtype, MTYPE_V, WN_store_offset(tree),
          pty_idx, WN_st(tree));
  swn = WN_CreateIntconst (OPC_U4INTCONST, size);
  wn  = WN_CreateMstore (0, pty_idx, WN_kid0(tree), awn, swn);
  WN_Set_Linenum(wn, current_srcpos);
  WN_set_field_id(wn, WN_field_id(tree));
  wn  = lower_store (block, wn, actions);

  WN_Delete(tree);
  return wn;
}


/* ====================================================================
 *
 * WN *lower_mistore(WN *block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on MISTORE nodes returning
 * an equivalent MSTORE node.
 *
 * ==================================================================== */

static WN *lower_mistore(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  TY_IDX pty_idx  = WN_ty(tree);
  TY_IDX ty_idx  = TY_pointed(pty_idx);

  if (WN_field_id (tree) != 0)
    ty_idx = get_field_type (ty_idx, WN_field_id (tree));
  
  UINT64 size    = TY_size(Ty_Table[ty_idx]);
  WN*    wn;
  WN*    swn;

  Is_True((WN_operator(tree) == OPR_ISTORE && WN_desc(tree) == MTYPE_M),
    ("expected mistore node, not %s", OPCODE_name(WN_opcode(tree))));
  Is_True(TY_kind(pty_idx) == KIND_POINTER,
    ("type specified in MISTORE not pointer"));
//Is_True(size > 0, ("type in MISTORE cannot be zero size"));
  if (size == 0)
    DevWarn ("type in MISTORE cannot be zero size");
  WN *rhs = WN_kid0(tree);
  if (WN_opcode(rhs) == OPC_MMLDID && WN_st(rhs) == Return_Val_Preg) {
    // handle lowering of MLDID of Return_Val_Preg followed by MISTORE
    Is_True(Action(LOWER_RETURN_VAL),
      ("LOWER_RETURN_VAL action must be specified"));
    return lower_return_mistore(block, tree, actions);
  }

  swn = WN_CreateIntconst(OPC_U4INTCONST, size);
  WN* kid0 = WN_COPY_Tree(WN_kid0(tree));
  WN* kid1 = WN_COPY_Tree(WN_kid1(tree));
  wn  = WN_CreateMstore(WN_offset(tree), pty_idx, kid0, kid1, swn);

  WN_Set_Linenum(wn, current_srcpos);
  WN_set_field_id(wn, WN_field_id(tree));
  set_original_wn (wn, tree);
  set_original_wn (kid0, WN_kid0(tree));
  wn  = lower_store(block, wn, actions);

  set_original_wn (wn, NULL); // avoid dangling pointer
  set_original_wn (WN_kid0(wn), NULL);
  WN_DELETE_Tree (tree);
  return wn;
}


#ifdef KEY
static BOOL Equiv (WN* wn1, WN* wn2) {
  if (!WN_Equiv(wn1,wn2)) return(FALSE);
  for (INT kidno=0; kidno<WN_kid_count(wn1); kidno++) {
    if (!Equiv(WN_kid(wn1,kidno),WN_kid(wn2,kidno))) {
      return(FALSE);
    }
  }
  return(TRUE);
}
#endif

#ifdef KEY // bug 12787
/* ====================================================================
 * walk expr looking for an INTRINSIC_OP that will become a function call;
 * return TRUE if found.
   ==================================================================== */
static BOOL has_call(WN *expr)
{
  if (WN_operator(expr) == OPR_INTRINSIC_OP &&
      ! INTRN_cg_intrinsic(WN_intrinsic(expr)))
    return TRUE;
  for (INT i = 0; i < WN_kid_count(expr); i++) {
    if (has_call(WN_kid(expr, i)))
      return TRUE;
  }
  return FALSE;
}
#endif

static BOOL Is_small_struct(WN* wn)
{
  Is_True((WN_opcode(WN_kid0(wn)) == OPC_MMILOAD || 
     WN_opcode(WN_kid0(wn)) == OPC_MMLDID), ("Expected M type loads"));

  if (TY_kind(WN_ty(WN_kid0(wn))) != KIND_STRUCT)
    return FALSE;

  if (TY_size(Ty_Table[WN_ty(WN_kid0(wn))]) < (Copy_Quantum_Ratio() * Pointer_Size +1)) {
      return TRUE;
  }
  return FALSE;
}


/* ====================================================================
 *
 * WN *lower_store(block, WN *tree, LOWER_ACTIONS actions)
 *
 * Perform lowering (see WN_Lower description) on store statement <tree>,
 * returning lowered tree.
 *
 * ==================================================================== */

static WN *lower_store(WN *block, WN *tree, LOWER_ACTIONS actions)
{
  BOOL kids_lowered = FALSE;  /* becomes TRUE when kids are lowered */

  Is_True(OPCODE_is_store(WN_opcode(tree)),
    ("expected store node, not %s", OPCODE_name(WN_opcode(tree))));

  /* If the store is to memory and the expression begins with a TAS, try to
   * replace with a store matching the result type of the TAS's expression.
   * Doing so may avoid a move from one register set to another.
   */
  if (Action(LOWER_TO_CG))
  {
    WN *tas = WN_kid0(tree);
    if (   WN_operator(tas) == OPR_TAS
  && (!WN_has_sym(tree) || WN_class(tree) != CLASS_PREG))
    {
      WN *tas_kid0 = WN_kid0(tas);
      TYPE_ID tas_kid0_rtype = WN_rtype(tas_kid0);
      if (MTYPE_byte_size(WN_rtype(tas)) == MTYPE_byte_size(tas_kid0_rtype)) {
  WN_set_desc(tree, tas_kid0_rtype);
  WN_kid0(tree) = tas_kid0;
  WN_Delete(tas);
      }
    }
  }

#if defined(KEY) && !defined(TARG_SL) // bug 11938
  // If OPT_Float_Via_Int is on, and both rhs and lhs are indirect memory ops,
  // change to use integer type for the copy
  if (Action(LOWER_TO_CG) && OPT_Float_Via_Int && ! WN_has_sym(tree)) {
    WN *rhs = WN_kid0(tree);
    if (MTYPE_is_float(WN_rtype(rhs)) && MTYPE_byte_size(WN_rtype(rhs)) <= 8 &&
        OPERATOR_is_load(WN_operator(rhs)) && ! WN_has_sym(rhs)) {
      TYPE_ID newtyp = Mtype_TransferSize(WN_rtype(rhs), MTYPE_I4);
      WN_set_rtype(rhs, newtyp);
      WN_set_desc(rhs, newtyp);
      WN_set_desc(tree, newtyp);
    }
  }
#endif

#ifdef TARG_X8664 // bug 11800
  // If the store is to memory and the expression is a load from memory, both
  // of MMX vector type, change to use integer types so it does not need to
  // involve MMX registers
  if (Action(LOWER_TO_CG) || Action(LOWER_BIT_FIELD_ID)) {
    WN *rhs = WN_kid0(tree);
    if (MTYPE_is_mmx_vector(WN_rtype(rhs)) &&
        OPERATOR_is_load(WN_operator(rhs))) {
      if (! (WN_has_sym(rhs) && WN_class(rhs) == CLASS_PREG) &&
          ! (WN_has_sym(tree) && WN_class(tree) == CLASS_PREG)) {
        TYPE_ID newtyp = Mtype_TransferSize(WN_rtype(rhs), MTYPE_I4);
        WN_set_rtype(rhs, newtyp);
        WN_set_desc(rhs, newtyp);
        WN_set_desc(tree, newtyp);
      }
    }
  }
#endif


  /*
   * create any maps that need to be present
   */
  lower_map(tree, actions);


  /* Note: We must split constant offsets after lowering complex stores
   * and splitting symbol addresses since these may create new offsets
   * that need to be split.
   */
  
  switch (WN_operator(tree))
  {
  case OPR_ISTORE:
    if (Action(LOWER_TO_MEMLIB) && WN_operator(tree) == OPR_ISTORE) {
        tree = Lower_Mistore_Memlib(block,tree,actions);
        if (tree == NULL || WN_operator(tree)!=OPR_ISTORE) 
           return tree; //otherwise continue
    } 
    if (WN_StoreIsUnused(WN_kid1(tree)))
    {
      WN  *eval;

      eval = lower_eval(block, WN_CreateEval(WN_kid0(tree)), actions);

      WN_Delete(tree);

      return eval;
    }

    if (Action(LOWER_MLDID_MSTID) && WN_desc(tree) == MTYPE_M)
      return lower_mistore(block, tree, actions);

    if (Action(LOWER_EARLY_MLOAD) && WN_desc(tree) == MTYPE_M){
      if ((WN_opcode(WN_kid0(tree)) == OPC_MMLDID || WN_opcode(WN_kid0(tree)) == OPC_MMILOAD) && Is_small_struct(tree)) {
  WN *return_block = lower_mistore_by_type(block, tree, actions);
  if (return_block) return return_block;
      }
    }

    if (Action(LOWER_RETURN_VAL) && WN_desc(tree) == MTYPE_M &&
       WN_opcode(WN_kid0(tree)) == OPC_MMLDID && 
       WN_st(WN_kid0(tree)) == Return_Val_Preg) {
       return lower_return_mistore(block, tree, actions);
    }
#ifdef TARG_X8664 // MMX vector return type also uses fake parameters (bug 7733)
    if (Action(LOWER_RETURN_VAL) && MTYPE_is_mmx_vector(WN_desc(tree)) &&
        WN_operator(WN_kid0(tree)) == OPR_LDID && 
        WN_st(WN_kid0(tree)) == Return_Val_Preg) {
       return lower_return_mistore(block, tree, actions);
    }
#endif


    if (Action(LOWER_BIT_FIELD_ID) && WN_desc(tree) == MTYPE_BS) {
      lower_bit_field_id(tree);
      if (Action(LOWER_BITS_OP) && WN_operator (tree) == OPR_ISTBITS)
  return lower_store_bits (block, tree, actions);
    }

    if (Action(LOWER_COMPLEX) && MTYPE_is_complex(WN_desc(tree)))
    {
      WN  *realstore, *imagstore;
      LEAF  realexpN, imagexpN;
      WN_OFFSET offset = WN_store_offset(tree);
      TYPE_ID realTY = Mtype_complex_to_real( WN_desc(tree));
      {
       /*
  * create the real/imaginary stores
  * load the temporary values into a preg before the store (pv314583)
  * as the store may interfere with the expression.
  */
  WN  *realexp, *imagexp;

  lower_complex_expr(block, WN_kid0(tree), actions, &realexp, &imagexp);
  
  realexpN = Make_Leaf(block, realexp, realTY);
  imagexpN = Make_Leaf(block, imagexp, realTY);
      }

      if (Action(LOWER_BASE_INDEX))
      {
  WN  *base, *index, *addr;
  LEAF  indexN;

  base = index= NULL;
  lower_to_base_index(WN_kid1(tree), &base, &index) ;

  base = lower_expr(block, base, actions);
  index = lower_expr(block, index, actions);

  indexN = Make_Leaf(block, index, Pointer_type);

  addr = WN_Add(Pointer_type,
          Load_Leaf(indexN),
          lower_copy_tree(base, actions));

  realstore = WN_Istore(realTY,
            offset,
            coerceTY(WN_ty(tree), realTY),
            addr,
            Load_Leaf(realexpN));

  addr = WN_Add(Pointer_type, Load_Leaf(indexN), base);

  imagstore = WN_Istore(realTY,
            coerceOFFSET(tree, realTY, offset),
            coerceTY(WN_ty(tree), realTY),
            addr,
            Load_Leaf(imagexpN));
      }
      else
      {
  WN  *addr;
        LEAF  addrN;

  addr = lower_expr(block, WN_kid1(tree), actions);
  addrN = Make_Leaf(block, addr, Pointer_type);

        realstore =  WN_Istore(realTY,
             offset,
             coerceTY(WN_ty(tree), realTY),
             Load_Leaf(addrN),
             Load_Leaf(realexpN));

        imagstore =  WN_Istore(realTY,
             coerceOFFSET(tree, realTY, offset),
             coerceTY(WN_ty(tree), realTY),
             Load_Leaf(addrN),
             Load_Leaf(imagexpN));
      }
      realstore = lower_store(block, realstore, actions);
      WN_Set_Linenum (realstore, WN_Get_Linenum(tree));
      WN_INSERT_BlockLast(block, realstore);

      imagstore = lower_store(block, imagstore, actions);
      lower_complex_maps(tree, realstore, imagstore, actions);
      WN_Delete(tree);

      return imagstore;
    }
    else if (Action(LOWER_QUAD) && MTYPE_is_quad(WN_desc(tree)))
    {
      WN  *hi, *lo, *hipart, *lopart;
      TYPE_ID realTY = MTYPE_F8;
      WN_OFFSET offset = WN_load_offset(tree);

      lower_quad_expr(block, WN_kid0(tree), actions, &hi, &lo);

      if (Action(