--- a/gcc/config/rs6000/sync.md	Sun Aug 21 07:07:55 2011 +0900
+++ b/gcc/config/rs6000/sync.md	Fri Oct 27 22:46:09 2017 +0900
@@ -1,5 +1,5 @@
 ;; Machine description for PowerPC synchronization instructions.
-;; Copyright (C) 2005, 2007, 2008, 2009 Free Software Foundation, Inc.
+;; Copyright (C) 2005-2017 Free Software Foundation, Inc.
 ;; Contributed by Geoffrey Keating.
 
 ;; This file is part of GCC.
@@ -18,21 +18,50 @@
 ;; along with GCC; see the file COPYING3.  If not see
 ;; <http://www.gnu.org/licenses/>.
 
-(define_mode_attr larx [(SI "lwarx") (DI "ldarx")])
-(define_mode_attr stcx [(SI "stwcx.") (DI "stdcx.")])
+(define_mode_attr larx [(QI "lbarx")
+			(HI "lharx")
+			(SI "lwarx")
+			(DI "ldarx")
+			(TI "lqarx")])
+
+(define_mode_attr stcx [(QI "stbcx.")
+			(HI "sthcx.")
+			(SI "stwcx.")
+			(DI "stdcx.")
+			(TI "stqcx.")])
 
 (define_code_iterator FETCHOP [plus minus ior xor and])
 (define_code_attr fetchop_name
-  [(plus "add") (minus "sub") (ior "ior") (xor "xor") (and "and")])
+  [(plus "add") (minus "sub") (ior "or") (xor "xor") (and "and")])
 (define_code_attr fetchop_pred
-  [(plus "add_operand") (minus "gpc_reg_operand")
+  [(plus "add_operand") (minus "int_reg_operand")
    (ior "logical_operand") (xor "logical_operand") (and "and_operand")])
-(define_code_attr fetchopsi_constr
-  [(plus "rIL") (minus "r") (ior "rKL") (xor "rKL") (and "rTKL")])
-(define_code_attr fetchopdi_constr
-  [(plus "rIL") (minus "r") (ior "rKJF") (xor "rKJF") (and "rSTKJ")])
 
-(define_expand "memory_barrier"
+(define_expand "mem_thread_fence"
+  [(match_operand:SI 0 "const_int_operand" "")]		;; model
+  ""
+{
+  enum memmodel model = memmodel_base (INTVAL (operands[0]));
+  switch (model)
+    {
+    case MEMMODEL_RELAXED:
+      break;
+    case MEMMODEL_CONSUME:
+    case MEMMODEL_ACQUIRE:
+    case MEMMODEL_RELEASE:
+    case MEMMODEL_ACQ_REL:
+      emit_insn (gen_lwsync ());
+      break;
+    case MEMMODEL_SEQ_CST:
+      emit_insn (gen_hwsync ());
+      break;
+    default:
+      gcc_unreachable ();
+    }
+  DONE;
+})
+
+(define_expand "hwsync"
   [(set (match_dup 0)
 	(unspec:BLK [(match_dup 0)] UNSPEC_SYNC))]
   ""
@@ -41,582 +70,415 @@
   MEM_VOLATILE_P (operands[0]) = 1;
 })
 
-(define_insn "*sync_internal"
+(define_insn "*hwsync"
   [(set (match_operand:BLK 0 "" "")
 	(unspec:BLK [(match_dup 0)] UNSPEC_SYNC))]
   ""
-  "{dcs|sync}"
+  "sync"
   [(set_attr "type" "sync")])
 
-(define_insn "load_locked_<mode>"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
-	(unspec_volatile:GPR
-	  [(match_operand:GPR 1 "memory_operand" "Z")] UNSPECV_LL))]
-  "TARGET_POWERPC"
-  "<larx> %0,%y1"
-  [(set_attr "type" "load_l")])
-
-(define_insn "store_conditional_<mode>"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x")
-	(unspec_volatile:CC [(const_int 0)] UNSPECV_SC))
-   (set (match_operand:GPR 1 "memory_operand" "=Z")
-	(match_operand:GPR 2 "gpc_reg_operand" "r"))]
-  "TARGET_POWERPC"
-  "<stcx> %2,%y1"
-  [(set_attr "type" "store_c")])
-
-(define_insn_and_split "sync_compare_and_swap<mode>"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=&r")
-	(match_operand:GPR 1 "memory_operand" "+Z"))
-   (set (match_dup 1)
-	(unspec:GPR
-	  [(match_operand:GPR 2 "reg_or_short_operand" "rI")
-	   (match_operand:GPR 3 "gpc_reg_operand" "r")]
-	  UNSPEC_CMPXCHG))
-   (clobber (match_scratch:GPR 4 "=&r"))
-   (clobber (match_scratch:CC 5 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
+(define_expand "lwsync"
+  [(set (match_dup 0)
+	(unspec:BLK [(match_dup 0)] UNSPEC_LWSYNC))]
+  ""
 {
-  rs6000_split_compare_and_swap (operands[0], operands[1], operands[2],
-				 operands[3], operands[4]);
-  DONE;
+  operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
+  MEM_VOLATILE_P (operands[0]) = 1;
 })
 
-(define_expand "sync_compare_and_swaphi"
-  [(match_operand:HI 0 "gpc_reg_operand" "")
-   (match_operand:HI 1 "memory_operand" "")
-   (match_operand:HI 2 "gpc_reg_operand" "")
-   (match_operand:HI 3 "gpc_reg_operand" "")]
-  "TARGET_POWERPC"
+(define_insn "*lwsync"
+  [(set (match_operand:BLK 0 "" "")
+	(unspec:BLK [(match_dup 0)] UNSPEC_LWSYNC))]
+  ""
 {
-  rs6000_expand_compare_and_swapqhi (operands[0], operands[1],
-				     operands[2], operands[3]);
-  DONE;
-})
+  /* Some AIX assemblers don't accept lwsync, so we use a .long.  */
+  if (TARGET_NO_LWSYNC)
+    return "sync";
+  else if (TARGET_LWSYNC_INSTRUCTION)
+    return "lwsync";
+  else
+    return ".long 0x7c2004ac";
+}
+  [(set_attr "type" "sync")])
+
+(define_insn "isync"
+  [(unspec_volatile:BLK [(const_int 0)] UNSPECV_ISYNC)]
+  ""
+  "isync"
+  [(set_attr "type" "isync")])
 
-(define_expand "sync_compare_and_swapqi"
-  [(match_operand:QI 0 "gpc_reg_operand" "")
-   (match_operand:QI 1 "memory_operand" "")
-   (match_operand:QI 2 "gpc_reg_operand" "")
-   (match_operand:QI 3 "gpc_reg_operand" "")]
-  "TARGET_POWERPC"
-{
-  rs6000_expand_compare_and_swapqhi (operands[0], operands[1],
-				     operands[2], operands[3]);
-  DONE;
-})
+;; Types that we should provide atomic instructions for.
+(define_mode_iterator AINT [QI
+			    HI
+			    SI
+			    (DI "TARGET_POWERPC64")
+			    (TI "TARGET_SYNC_TI")])
+
+;; The control dependency used for load dependency described
+;; in B.2.3 of the Power ISA 2.06B.
+(define_insn "loadsync_<mode>"
+  [(unspec_volatile:BLK [(match_operand:AINT 0 "register_operand" "r")]
+			UNSPECV_ISYNC)
+   (clobber (match_scratch:CC 1 "=y"))]
+  ""
+  "cmpw %1,%0,%0\;bne- %1,$+4\;isync"
+  [(set_attr "type" "isync")
+   (set_attr "length" "12")])
+
+(define_insn "load_quadpti"
+  [(set (match_operand:PTI 0 "quad_int_reg_operand" "=&r")
+	(unspec:PTI
+	 [(match_operand:TI 1 "quad_memory_operand" "wQ")] UNSPEC_LSQ))]
+  "TARGET_SYNC_TI
+   && !reg_mentioned_p (operands[0], operands[1])"
+  "lq %0,%1"
+  [(set_attr "type" "load")
+   (set_attr "length" "4")])
 
-(define_insn_and_split "sync_compare_and_swapqhi_internal"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=&r")
-	(match_operand:SI 4 "memory_operand" "+Z"))
-   (set (match_dup 4)
-        (unspec:SI
-          [(match_operand:SI 1 "gpc_reg_operand" "r")
-           (match_operand:SI 2 "gpc_reg_operand" "r")
-           (match_operand:SI 3 "gpc_reg_operand" "r")]
-          UNSPEC_CMPXCHG))
-   (clobber (match_scratch:SI 5 "=&r"))
-   (clobber (match_scratch:CC 6 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
+(define_expand "atomic_load<mode>"
+  [(set (match_operand:AINT 0 "register_operand" "")		;; output
+	(match_operand:AINT 1 "memory_operand" ""))		;; memory
+   (use (match_operand:SI 2 "const_int_operand" ""))]		;; model
+  ""
 {
-  rs6000_split_compare_and_swapqhi (operands[0], operands[1],
-				    operands[2], operands[3], operands[4],
-				    operands[5]);
-  DONE;
-})
+  if (<MODE>mode == TImode && !TARGET_SYNC_TI)
+    FAIL;
+
+  enum memmodel model = memmodel_base (INTVAL (operands[2]));
+
+  if (is_mm_seq_cst (model))
+    emit_insn (gen_hwsync ());
+
+  if (<MODE>mode != TImode)
+    emit_move_insn (operands[0], operands[1]);
+  else
+    {
+      rtx op0 = operands[0];
+      rtx op1 = operands[1];
+      rtx pti_reg = gen_reg_rtx (PTImode);
+
+      if (!quad_address_p (XEXP (op1, 0), TImode, false))
+	{
+	  rtx old_addr = XEXP (op1, 0);
+	  rtx new_addr = force_reg (Pmode, old_addr);
+	  operands[1] = op1 = replace_equiv_address (op1, new_addr);
+	}
 
-(define_insn_and_split "sync_lock_test_and_set<mode>"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=&r")
-	(match_operand:GPR 1 "memory_operand" "+Z"))
-   (set (match_dup 1)
-	(unspec:GPR
-	  [(match_operand:GPR 2 "reg_or_short_operand" "rL")]
-	  UNSPEC_XCHG))
-   (clobber (match_scratch:GPR 3 "=&r"))
-   (clobber (match_scratch:CC 4 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rs6000_split_lock_test_and_set (operands[0], operands[1], operands[2],
-				  operands[3]);
+      emit_insn (gen_load_quadpti (pti_reg, op1));
+
+      if (WORDS_BIG_ENDIAN)
+	emit_move_insn (op0, gen_lowpart (TImode, pti_reg));
+      else
+	{
+	  emit_move_insn (gen_lowpart (DImode, op0), gen_highpart (DImode, pti_reg));
+	  emit_move_insn (gen_highpart (DImode, op0), gen_lowpart (DImode, pti_reg));
+	}
+    }
+
+  switch (model)
+    {
+    case MEMMODEL_RELAXED:
+      break;
+    case MEMMODEL_CONSUME:
+    case MEMMODEL_ACQUIRE:
+    case MEMMODEL_SEQ_CST:
+      emit_insn (gen_loadsync_<mode> (operands[0]));
+      break;
+    default:
+      gcc_unreachable ();
+    }
   DONE;
 })
 
-(define_expand "sync_<fetchop_name><mode>"
-  [(parallel [(set (match_operand:INT1 0 "memory_operand" "")
-		   (unspec:INT1
-		     [(FETCHOP:INT1 (match_dup 0)
-			(match_operand:INT1 1 "<fetchop_pred>" ""))]
-		     UNSPEC_ATOMIC))
-	      (clobber (scratch:INT1))
-	      (clobber (scratch:CC))])]
-  "TARGET_POWERPC"
-  "
-{
-  if (<MODE>mode != SImode && <MODE>mode != DImode)
-    {
-      if (PPC405_ERRATUM77)
-	FAIL;
-      rs6000_emit_sync (<CODE>, <MODE>mode, operands[0], operands[1],
-			NULL_RTX, NULL_RTX, true);
-      DONE;
-    }
-}")
-
-(define_insn_and_split "*sync_<fetchop_name>si_internal"
-  [(set (match_operand:SI 0 "memory_operand" "+Z")
-	(unspec:SI
-	  [(FETCHOP:SI (match_dup 0)
-	     (match_operand:SI 1 "<fetchop_pred>" "<fetchopsi_constr>"))]
-	  UNSPEC_ATOMIC))
-   (clobber (match_scratch:SI 2 "=&b"))
-   (clobber (match_scratch:CC 3 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rs6000_split_atomic_op (<CODE>, operands[0], operands[1],
-			  NULL_RTX, NULL_RTX, operands[2]);
-  DONE;
-})
+(define_insn "store_quadpti"
+  [(set (match_operand:PTI 0 "quad_memory_operand" "=wQ")
+	(unspec:PTI
+	 [(match_operand:PTI 1 "quad_int_reg_operand" "r")] UNSPEC_LSQ))]
+  "TARGET_SYNC_TI"
+  "stq %1,%0"
+  [(set_attr "type" "store")
+   (set_attr "length" "4")])
 
-(define_insn_and_split "*sync_<fetchop_name>di_internal"
-  [(set (match_operand:DI 0 "memory_operand" "+Z")
-	(unspec:DI
-	  [(FETCHOP:DI (match_dup 0)
-	     (match_operand:DI 1 "<fetchop_pred>" "<fetchopdi_constr>"))]
-	  UNSPEC_ATOMIC))
-   (clobber (match_scratch:DI 2 "=&b"))
-   (clobber (match_scratch:CC 3 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
+(define_expand "atomic_store<mode>"
+  [(set (match_operand:AINT 0 "memory_operand" "")		;; memory
+	(match_operand:AINT 1 "register_operand" ""))		;; input
+   (use (match_operand:SI 2 "const_int_operand" ""))]		;; model
+  ""
 {
-  rs6000_split_atomic_op (<CODE>, operands[0], operands[1],
-			  NULL_RTX, NULL_RTX, operands[2]);
-  DONE;
-})
-
-(define_expand "sync_nand<mode>"
-  [(parallel [(set (match_operand:INT1 0 "memory_operand" "")
-	      (unspec:INT1
-		[(ior:INT1 (not:INT1 (match_dup 0))
-			   (not:INT1 (match_operand:INT1 1 "gpc_reg_operand" "")))]
-		UNSPEC_ATOMIC))
-	      (clobber (scratch:INT1))
-	      (clobber (scratch:CC))])]
-  "TARGET_POWERPC"
-  "
-{
-  if (<MODE>mode != SImode && <MODE>mode != DImode)
-    {
-      FAIL;
-      if (PPC405_ERRATUM77)
-	FAIL;
-      rs6000_emit_sync (NOT, <MODE>mode, operands[0], operands[1],
-			NULL_RTX, NULL_RTX, true);
-      DONE;
-    }
-}")
+  if (<MODE>mode == TImode && !TARGET_SYNC_TI)
+    FAIL;
 
-(define_insn_and_split "*sync_nand<mode>_internal"
-  [(set (match_operand:GPR 0 "memory_operand" "+Z")
-	(unspec:GPR
-	  [(ior:GPR (not:GPR (match_dup 0))
-		    (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
-	  UNSPEC_ATOMIC))
-   (clobber (match_scratch:GPR 2 "=&r"))
-   (clobber (match_scratch:CC 3 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rs6000_split_atomic_op (NOT, operands[0], operands[1],
-			  NULL_RTX, NULL_RTX, operands[2]);
-  DONE;
-})
-
-(define_expand "sync_old_<fetchop_name><mode>"
-  [(parallel [(set (match_operand:INT1 0 "gpc_reg_operand" "")
-		   (match_operand:INT1 1 "memory_operand" ""))
-	      (set (match_dup 1)
-		   (unspec:INT1
-		     [(FETCHOP:INT1 (match_dup 1)
-			(match_operand:INT1 2 "<fetchop_pred>" ""))]
-		     UNSPEC_ATOMIC))
-	      (clobber (scratch:INT1))
-	      (clobber (scratch:CC))])]
-  "TARGET_POWERPC"
-  "
-{ 
-  if (<MODE>mode != SImode && <MODE>mode != DImode)
+  enum memmodel model = memmodel_base (INTVAL (operands[2]));
+  switch (model)
     {
-      if (PPC405_ERRATUM77)
-	FAIL;
-      rs6000_emit_sync (<CODE>, <MODE>mode, operands[1], operands[2],
-			operands[0], NULL_RTX, true);
-      DONE;
+    case MEMMODEL_RELAXED:
+      break;
+    case MEMMODEL_RELEASE:
+      emit_insn (gen_lwsync ());
+      break;
+    case MEMMODEL_SEQ_CST:
+      emit_insn (gen_hwsync ());
+      break;
+    default:
+      gcc_unreachable ();
     }
-}")
+  if (<MODE>mode != TImode)
+    emit_move_insn (operands[0], operands[1]);
+  else
+    {
+      rtx op0 = operands[0];
+      rtx op1 = operands[1];
+      rtx pti_reg = gen_reg_rtx (PTImode);
 
-(define_insn_and_split "*sync_old_<fetchop_name>si_internal"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=&r")
-	(match_operand:SI 1 "memory_operand" "+Z"))
-   (set (match_dup 1)
-	(unspec:SI
-	  [(FETCHOP:SI (match_dup 1)
-	     (match_operand:SI 2 "<fetchop_pred>" "<fetchopsi_constr>"))]
-	  UNSPEC_ATOMIC))
-   (clobber (match_scratch:SI 3 "=&b"))
-   (clobber (match_scratch:CC 4 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rs6000_split_atomic_op (<CODE>, operands[1], operands[2],
-			  operands[0], NULL_RTX, operands[3]);
-  DONE;
-})
+      if (!quad_address_p (XEXP (op0, 0), TImode, false))
+	{
+	  rtx old_addr = XEXP (op0, 0);
+	  rtx new_addr = force_reg (Pmode, old_addr);
+	  operands[0] = op0 = replace_equiv_address (op0, new_addr);
+	}
 
-(define_insn_and_split "*sync_old_<fetchop_name>di_internal"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r")
-	(match_operand:DI 1 "memory_operand" "+Z"))
-   (set (match_dup 1)
-	(unspec:DI
-	  [(FETCHOP:DI (match_dup 1)
-	     (match_operand:DI 2 "<fetchop_pred>" "<fetchopdi_constr>"))]
-	  UNSPEC_ATOMIC))
-   (clobber (match_scratch:DI 3 "=&b"))
-   (clobber (match_scratch:CC 4 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rs6000_split_atomic_op (<CODE>, operands[1], operands[2],
-			  operands[0], NULL_RTX, operands[3]);
+      if (WORDS_BIG_ENDIAN)
+	emit_move_insn (pti_reg, gen_lowpart (PTImode, op1));
+      else
+	{
+	  emit_move_insn (gen_lowpart (DImode, pti_reg), gen_highpart (DImode, op1));
+	  emit_move_insn (gen_highpart (DImode, pti_reg), gen_lowpart (DImode, op1));
+	}
+
+      emit_insn (gen_store_quadpti (gen_lowpart (PTImode, op0), pti_reg));
+    }
+
   DONE;
 })
 
-(define_expand "sync_old_nand<mode>"
-  [(parallel [(set (match_operand:INT1 0 "gpc_reg_operand" "")
-		   (match_operand:INT1 1 "memory_operand" ""))
-	      (set (match_dup 1)
-		   (unspec:INT1
-		     [(ior:INT1 (not:INT1 (match_dup 1))
-				(not:INT1 (match_operand:INT1 2 "gpc_reg_operand" "")))]
-		     UNSPEC_ATOMIC))
-	      (clobber (scratch:INT1))
-	      (clobber (scratch:CC))])]
-  "TARGET_POWERPC"
-  "
-{
-  if (<MODE>mode != SImode && <MODE>mode != DImode)
-    {
-      FAIL;
-      if (PPC405_ERRATUM77)
-	FAIL;
-      rs6000_emit_sync (NOT, <MODE>mode, operands[1], operands[2],
-			operands[0], NULL_RTX, true);
-      DONE;
-    }
-}")
+;; Any supported integer mode that has atomic l<x>arx/st<x>cx. instrucitons
+;; other than the quad memory operations, which have special restrictions.
+;; Byte/halfword atomic instructions were added in ISA 2.06B, but were phased
+;; in and did not show up until power8.  TImode atomic lqarx/stqcx. require
+;; special handling due to even/odd register requirements.
+(define_mode_iterator ATOMIC [(QI "TARGET_SYNC_HI_QI")
+			      (HI "TARGET_SYNC_HI_QI")
+			      SI
+			      (DI "TARGET_POWERPC64")])
 
-(define_insn_and_split "*sync_old_nand<mode>_internal"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=&r")
-	(match_operand:GPR 1 "memory_operand" "+Z"))
-   (set (match_dup 1)
-	(unspec:GPR
-	  [(ior:GPR (not:GPR (match_dup 1))
-		    (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r")))]
-	  UNSPEC_ATOMIC))
-   (clobber (match_scratch:GPR 3 "=&r"))
-   (clobber (match_scratch:CC 4 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rs6000_split_atomic_op (NOT, operands[1], operands[2],
-			  operands[0], NULL_RTX, operands[3]);
-  DONE;
-})
+(define_insn "load_locked<mode>"
+  [(set (match_operand:ATOMIC 0 "int_reg_operand" "=r")
+	(unspec_volatile:ATOMIC
+         [(match_operand:ATOMIC 1 "memory_operand" "Z")] UNSPECV_LL))]
+  ""
+  "<larx> %0,%y1"
+  [(set_attr "type" "load_l")])
+
+(define_insn "load_locked<QHI:mode>_si"
+  [(set (match_operand:SI 0 "int_reg_operand" "=r")
+	(unspec_volatile:SI
+	  [(match_operand:QHI 1 "memory_operand" "Z")] UNSPECV_LL))]
+  "TARGET_SYNC_HI_QI"
+  "<QHI:larx> %0,%y1"
+  [(set_attr "type" "load_l")])
 
-(define_expand "sync_new_<fetchop_name><mode>"
-  [(parallel [(set (match_operand:INT1 0 "gpc_reg_operand" "")
-		   (FETCHOP:INT1
-		     (match_operand:INT1 1 "memory_operand" "")
-		     (match_operand:INT1 2 "<fetchop_pred>" "")))
-	      (set (match_dup 1)
-		   (unspec:INT1
-		     [(FETCHOP:INT1 (match_dup 1) (match_dup 2))]
-		     UNSPEC_ATOMIC))
-	      (clobber (scratch:INT1))
-	      (clobber (scratch:CC))])]
-  "TARGET_POWERPC"
-  "
-{
-  if (<MODE>mode != SImode && <MODE>mode != DImode)
-    {
-      if (PPC405_ERRATUM77)
-	FAIL;
-      rs6000_emit_sync (<CODE>, <MODE>mode, operands[1], operands[2],
-			NULL_RTX, operands[0], true);
-      DONE;
-    }
-}")
+;; Use PTImode to get even/odd register pairs.
+;; Use a temporary register to force getting an even register for the
+;; lqarx/stqcrx. instructions.  Normal optimizations will eliminate this extra
+;; copy on big endian systems.
+
+;; On little endian systems where non-atomic quad word load/store instructions
+;; are not used, the address can be register+offset, so make sure the address
+;; is indexed or indirect before register allocation.
 
-(define_insn_and_split "*sync_new_<fetchop_name>si_internal"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=&r")
-	(FETCHOP:SI
-	  (match_operand:SI 1 "memory_operand" "+Z")
-	  (match_operand:SI 2 "<fetchop_pred>" "<fetchopsi_constr>")))
-   (set (match_dup 1)
-	(unspec:SI
-	  [(FETCHOP:SI (match_dup 1) (match_dup 2))]
-	  UNSPEC_ATOMIC))
-   (clobber (match_scratch:SI 3 "=&b"))
-   (clobber (match_scratch:CC 4 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
+(define_expand "load_lockedti"
+  [(use (match_operand:TI 0 "quad_int_reg_operand" ""))
+   (use (match_operand:TI 1 "memory_operand" ""))]
+  "TARGET_SYNC_TI"
 {
-  rs6000_split_atomic_op (<CODE>, operands[1], operands[2],
-			  NULL_RTX, operands[0], operands[3]);
-  DONE;
-})
+  rtx op0 = operands[0];
+  rtx op1 = operands[1];
+  rtx pti = gen_reg_rtx (PTImode);
 
-(define_insn_and_split "*sync_new_<fetchop_name>di_internal"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r")
-	(FETCHOP:DI
-	  (match_operand:DI 1 "memory_operand" "+Z")
-	  (match_operand:DI 2 "<fetchop_pred>" "<fetchopdi_constr>")))
-   (set (match_dup 1)
-	(unspec:DI
-	  [(FETCHOP:DI (match_dup 1) (match_dup 2))]
-	  UNSPEC_ATOMIC))
-   (clobber (match_scratch:DI 3 "=&b"))
-   (clobber (match_scratch:CC 4 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rs6000_split_atomic_op (<CODE>, operands[1], operands[2],
-			  NULL_RTX, operands[0], operands[3]);
+  if (!indexed_or_indirect_operand (op1, TImode))
+    {
+      rtx old_addr = XEXP (op1, 0);
+      rtx new_addr = force_reg (Pmode, old_addr);
+      operands[1] = op1 = change_address (op1, TImode, new_addr);
+    }
+
+  emit_insn (gen_load_lockedpti (pti, op1));
+  if (WORDS_BIG_ENDIAN)
+    emit_move_insn (op0, gen_lowpart (TImode, pti));
+  else
+    {
+      emit_move_insn (gen_lowpart (DImode, op0), gen_highpart (DImode, pti));
+      emit_move_insn (gen_highpart (DImode, op0), gen_lowpart (DImode, pti));
+    }
   DONE;
 })
 
-(define_expand "sync_new_nand<mode>"
-  [(parallel [(set (match_operand:INT1 0 "gpc_reg_operand" "")
-		   (ior:INT1
-		     (not:INT1 (match_operand:INT1 1 "memory_operand" ""))
-		     (not:INT1 (match_operand:INT1 2 "gpc_reg_operand" ""))))
-	      (set (match_dup 1)
-		   (unspec:INT1
-		     [(ior:INT1 (not:INT1 (match_dup 1))
-				(not:INT1 (match_dup 2)))]
-		     UNSPEC_ATOMIC))
-	      (clobber (scratch:INT1))
-	      (clobber (scratch:CC))])]
-  "TARGET_POWERPC"
-  "
+(define_insn "load_lockedpti"
+  [(set (match_operand:PTI 0 "quad_int_reg_operand" "=&r")
+	(unspec_volatile:PTI
+         [(match_operand:TI 1 "indexed_or_indirect_operand" "Z")] UNSPECV_LL))]
+  "TARGET_SYNC_TI
+   && !reg_mentioned_p (operands[0], operands[1])
+   && quad_int_reg_operand (operands[0], PTImode)"
+  "lqarx %0,%y1"
+  [(set_attr "type" "load_l")])
+
+(define_insn "store_conditional<mode>"
+  [(set (match_operand:CC 0 "cc_reg_operand" "=x")
+	(unspec_volatile:CC [(const_int 0)] UNSPECV_SC))
+   (set (match_operand:ATOMIC 1 "memory_operand" "=Z")
+	(match_operand:ATOMIC 2 "int_reg_operand" "r"))]
+  ""
+  "<stcx> %2,%y1"
+  [(set_attr "type" "store_c")])
+
+;; Use a temporary register to force getting an even register for the
+;; lqarx/stqcrx. instructions.  Normal optimizations will eliminate this extra
+;; copy on big endian systems.
+
+;; On little endian systems where non-atomic quad word load/store instructions
+;; are not used, the address can be register+offset, so make sure the address
+;; is indexed or indirect before register allocation.
+
+(define_expand "store_conditionalti"
+  [(use (match_operand:CC 0 "cc_reg_operand" ""))
+   (use (match_operand:TI 1 "memory_operand" ""))
+   (use (match_operand:TI 2 "quad_int_reg_operand" ""))]
+  "TARGET_SYNC_TI"
 {
-  if (<MODE>mode != SImode && <MODE>mode != DImode)
+  rtx op0 = operands[0];
+  rtx op1 = operands[1];
+  rtx op2 = operands[2];
+  rtx addr = XEXP (op1, 0);
+  rtx pti_mem;
+  rtx pti_reg;
+
+  if (!indexed_or_indirect_operand (op1, TImode))
     {
-      FAIL;
-      if (PPC405_ERRATUM77)
-	FAIL;
-      rs6000_emit_sync (NOT, <MODE>mode, operands[1], operands[2],
-			NULL_RTX, operands[0], true);
-      DONE;
+      rtx new_addr = force_reg (Pmode, addr);
+      operands[1] = op1 = change_address (op1, TImode, new_addr);
+      addr = new_addr;
     }
-}")
+
+  pti_mem = change_address (op1, PTImode, addr);
+  pti_reg = gen_reg_rtx (PTImode);
 
-(define_insn_and_split "*sync_new_nand<mode>_internal"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=&r")
-	(ior:GPR
-	  (not:GPR (match_operand:GPR 1 "memory_operand" "+Z"))
-	  (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r"))))
-   (set (match_dup 1)
-	(unspec:GPR
-	  [(ior:GPR (not:GPR (match_dup 1)) (not:GPR (match_dup 2)))]
-	  UNSPEC_ATOMIC))
-   (clobber (match_scratch:GPR 3 "=&r"))
-   (clobber (match_scratch:CC 4 "=&x"))]
-  "TARGET_POWERPC"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rs6000_split_atomic_op (NOT, operands[1], operands[2],
-			  NULL_RTX, operands[0], operands[3]);
+  if (WORDS_BIG_ENDIAN)
+    emit_move_insn (pti_reg, gen_lowpart (PTImode, op2));
+  else
+    {
+      emit_move_insn (gen_lowpart (DImode, pti_reg), gen_highpart (DImode, op2));
+      emit_move_insn (gen_highpart (DImode, pti_reg), gen_lowpart (DImode, op2));
+    }
+
+  emit_insn (gen_store_conditionalpti (op0, pti_mem, pti_reg));
   DONE;
 })
 
-; and<mode> without cr0 clobber to avoid generation of additional clobber 
-; in atomic splitters causing internal consistency failure.
-; cr0 already clobbered by larx/stcx.
-(define_insn "*atomic_andsi"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-	(unspec:SI [(match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r")
-		    (match_operand:SI 2 "and_operand" "?r,T,K,L")]
-		    UNSPEC_AND))]
-  ""
-  "@
-   and %0,%1,%2
-   {rlinm|rlwinm} %0,%1,0,%m2,%M2
-   {andil.|andi.} %0,%1,%b2
-   {andiu.|andis.} %0,%1,%u2"
-  [(set_attr "type" "*,*,compare,compare")])
+(define_insn "store_conditionalpti"
+  [(set (match_operand:CC 0 "cc_reg_operand" "=x")
+	(unspec_volatile:CC [(const_int 0)] UNSPECV_SC))
+   (set (match_operand:PTI 1 "indexed_or_indirect_operand" "=Z")
+	(match_operand:PTI 2 "quad_int_reg_operand" "r"))]
+  "TARGET_SYNC_TI && quad_int_reg_operand (operands[2], PTImode)"
+  "stqcx. %2,%y1"
+  [(set_attr "type" "store_c")])
 
-(define_insn "*atomic_anddi"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r,r")
-	(unspec:DI [(match_operand:DI 1 "gpc_reg_operand" "%r,r,r,r,r")
-		    (match_operand:DI 2 "and_operand" "?r,S,T,K,J")]
-		    UNSPEC_AND))]
-  "TARGET_POWERPC64"
-  "@
-   and %0,%1,%2
-   rldic%B2 %0,%1,0,%S2
-   rlwinm %0,%1,0,%m2,%M2
-   andi. %0,%1,%b2
-   andis. %0,%1,%u2"
-  [(set_attr "type" "*,*,*,compare,compare")
-   (set_attr "length" "4,4,4,4,4")])
-
-; the sync_*_internal patterns all have these operands:
-; 0 - memory location
-; 1 - operand
-; 2 - value in memory after operation
-; 3 - value in memory immediately before operation
-
-(define_insn "*sync_addshort_internal"
-  [(set (match_operand:SI 2 "gpc_reg_operand" "=&r")
-	(ior:SI (and:SI (plus:SI (match_operand:SI 0 "memory_operand" "+Z")
-				 (match_operand:SI 1 "add_operand" "rI"))
-			(match_operand:SI 4 "gpc_reg_operand" "r"))
-		(and:SI (not:SI (match_dup 4)) (match_dup 0))))
-   (set (match_operand:SI 3 "gpc_reg_operand" "=&b") (match_dup 0))
-   (set (match_dup 0)
-	(unspec:SI [(ior:SI (and:SI (plus:SI (match_dup 0) (match_dup 1))
-				    (match_dup 4))
-			    (and:SI (not:SI (match_dup 4)) (match_dup 0)))]
-		   UNSPEC_SYNC_OP))
-   (clobber (match_scratch:CC 5 "=&x"))
-   (clobber (match_scratch:SI 6 "=&r"))]
-  "TARGET_POWERPC && !PPC405_ERRATUM77"
-  "lwarx %3,%y0\n\tadd%I1 %2,%3,%1\n\tandc %6,%3,%4\n\tand %2,%2,%4\n\tor %2,%2,%6\n\tstwcx. %2,%y0\n\tbne- $-24"
-  [(set_attr "length" "28")])
+(define_expand "atomic_compare_and_swap<mode>"
+  [(match_operand:SI 0 "int_reg_operand" "")		;; bool out
+   (match_operand:AINT 1 "int_reg_operand" "")		;; val out
+   (match_operand:AINT 2 "memory_operand" "")		;; memory
+   (match_operand:AINT 3 "reg_or_short_operand" "")	;; expected
+   (match_operand:AINT 4 "int_reg_operand" "")		;; desired
+   (match_operand:SI 5 "const_int_operand" "")		;; is_weak
+   (match_operand:SI 6 "const_int_operand" "")		;; model succ
+   (match_operand:SI 7 "const_int_operand" "")]		;; model fail
+  ""
+{
+  rs6000_expand_atomic_compare_and_swap (operands);
+  DONE;
+})
 
-(define_insn "*sync_subshort_internal"
-  [(set (match_operand:SI 2 "gpc_reg_operand" "=&r")
-	(ior:SI (and:SI (minus:SI (match_operand:SI 0 "memory_operand" "+Z")
-				  (match_operand:SI 1 "add_operand" "rI"))
-			(match_operand:SI 4 "gpc_reg_operand" "r"))
-		(and:SI (not:SI (match_dup 4)) (match_dup 0))))
-   (set (match_operand:SI 3 "gpc_reg_operand" "=&b") (match_dup 0))
-   (set (match_dup 0)
-	(unspec:SI [(ior:SI (and:SI (minus:SI (match_dup 0) (match_dup 1))
-				    (match_dup 4))
-			    (and:SI (not:SI (match_dup 4)) (match_dup 0)))]
-		   UNSPEC_SYNC_OP))
-   (clobber (match_scratch:CC 5 "=&x"))
-   (clobber (match_scratch:SI 6 "=&r"))]
-  "TARGET_POWERPC && !PPC405_ERRATUM77"
-  "lwarx %3,%y0\n\tsubf %2,%1,%3\n\tandc %6,%3,%4\n\tand %2,%2,%4\n\tor %2,%2,%6\n\tstwcx. %2,%y0\n\tbne- $-24"
-  [(set_attr "length" "28")])
-
-(define_insn "*sync_andsi_internal"
-  [(set (match_operand:SI 2 "gpc_reg_operand" "=&r,&r,&r,&r")
-	(and:SI (match_operand:SI 0 "memory_operand" "+Z,Z,Z,Z")
-		(match_operand:SI 1 "and_operand" "r,T,K,L")))
-   (set (match_operand:SI 3 "gpc_reg_operand" "=&b,&b,&b,&b") (match_dup 0))
-   (set (match_dup 0)
-	(unspec:SI [(and:SI (match_dup 0) (match_dup 1))]
-		   UNSPEC_SYNC_OP))
-   (clobber (match_scratch:CC 4 "=&x,&x,&x,&x"))]
-  "TARGET_POWERPC && !PPC405_ERRATUM77"
-  "@
-   lwarx %3,%y0\n\tand %2,%3,%1\n\tstwcx. %2,%y0\n\tbne- $-12
-   lwarx %3,%y0\n\trlwinm %2,%3,0,%m1,%M1\n\tstwcx. %2,%y0\n\tbne- $-12
-   lwarx %3,%y0\n\tandi. %2,%3,%b1\n\tstwcx. %2,%y0\n\tbne- $-12
-   lwarx %3,%y0\n\tandis. %2,%3,%u1\n\tstwcx. %2,%y0\n\tbne- $-12"
-  [(set_attr "length" "16,16,16,16")])
+(define_expand "atomic_exchange<mode>"
+  [(match_operand:AINT 0 "int_reg_operand" "")		;; output
+   (match_operand:AINT 1 "memory_operand" "")		;; memory
+   (match_operand:AINT 2 "int_reg_operand" "")		;; input
+   (match_operand:SI 3 "const_int_operand" "")]		;; model
+  ""
+{
+  rs6000_expand_atomic_exchange (operands);
+  DONE;
+})
 
-(define_insn "*sync_boolsi_internal"
-  [(set (match_operand:SI 2 "gpc_reg_operand" "=&r,&r,&r")
-	(match_operator:SI 4 "boolean_or_operator"
-	 [(match_operand:SI 0 "memory_operand" "+Z,Z,Z")
-	  (match_operand:SI 1 "logical_operand" "r,K,L")]))
-   (set (match_operand:SI 3 "gpc_reg_operand" "=&b,&b,&b") (match_dup 0))
-   (set (match_dup 0) (unspec:SI [(match_dup 4)] UNSPEC_SYNC_OP))
-   (clobber (match_scratch:CC 5 "=&x,&x,&x"))]
-  "TARGET_POWERPC && !PPC405_ERRATUM77"
-  "@
-   lwarx %3,%y0\n\t%q4 %2,%3,%1\n\tstwcx. %2,%y0\n\tbne- $-12
-   lwarx %3,%y0\n\t%q4i %2,%3,%b1\n\tstwcx. %2,%y0\n\tbne- $-12
-   lwarx %3,%y0\n\t%q4is %2,%3,%u1\n\tstwcx. %2,%y0\n\tbne- $-12"
-  [(set_attr "length" "16,16,16")])
+(define_expand "atomic_<fetchop_name><mode>"
+  [(match_operand:AINT 0 "memory_operand" "")		;; memory
+   (FETCHOP:AINT (match_dup 0)
+     (match_operand:AINT 1 "<fetchop_pred>" ""))	;; operand
+   (match_operand:SI 2 "const_int_operand" "")]		;; model
+  ""
+{
+  rs6000_expand_atomic_op (<CODE>, operands[0], operands[1],
+			   NULL_RTX, NULL_RTX, operands[2]);
+  DONE;
+})
 
-; This pattern could also take immediate values of operand 1,
-; since the non-NOT version of the operator is used; but this is not
-; very useful, since in practice operand 1 is a full 32-bit value.
-; Likewise, operand 5 is in practice either <= 2^16 or it is a register.
-(define_insn "*sync_boolcshort_internal"
-  [(set (match_operand:SI 2 "gpc_reg_operand" "=&r")
-	(match_operator:SI 4 "boolean_or_operator"
-	 [(xor:SI (not:SI (match_operand:SI 0 "memory_operand" "+Z"))
-		  (not:SI (match_operand:SI 5 "logical_operand" "rK")))
-	 (match_operand:SI 1 "gpc_reg_operand" "r")]))
-   (set (match_operand:SI 3 "gpc_reg_operand" "=&b") (match_dup 0))
-   (set (match_dup 0) (unspec:SI [(match_dup 4)] UNSPEC_SYNC_OP))
-   (clobber (match_scratch:CC 6 "=&x"))]
-  "TARGET_POWERPC && !PPC405_ERRATUM77"
-  "lwarx %3,%y0\n\txor%I2 %2,%3,%5\n\t%q4 %2,%2,%1\n\tstwcx. %2,%y0\n\tbne- $-16"
-  [(set_attr "length" "20")])
-
-(define_insn "isync"
-  [(set (mem:BLK (match_scratch 0 "X"))
-	(unspec_volatile:BLK [(mem:BLK (match_scratch 1 "X"))] UNSPEC_ISYNC))]
-  ""
-  "{ics|isync}"
-  [(set_attr "type" "isync")])
-
-(define_expand "sync_lock_release<mode>"
-  [(set (match_operand:INT 0 "memory_operand")
-	(match_operand:INT 1 "any_operand"))]
-  ""
-  "
-{
-  emit_insn (gen_lwsync ());
-  emit_move_insn (operands[0], operands[1]);
-  DONE;
-}")
-
-; Some AIX assemblers don't accept lwsync, so we use a .long.
-(define_insn "lwsync"
-  [(set (mem:BLK (match_scratch 0 "X"))
-	(unspec_volatile:BLK [(mem:BLK (match_scratch 1 "X"))] UNSPEC_LWSYNC))]
+(define_expand "atomic_nand<mode>"
+  [(match_operand:AINT 0 "memory_operand" "")		;; memory
+   (match_operand:AINT 1 "int_reg_operand" "")		;; operand
+   (match_operand:SI 2 "const_int_operand" "")]		;; model
   ""
 {
-  if (TARGET_NO_LWSYNC)
-    return "sync";
-  else
-    return (TARGET_LWSYNC_INSTRUCTION) ? "lwsync" : ".long 0x7c2004ac";
-}
-  [(set_attr "type" "sync")])
+  rs6000_expand_atomic_op (NOT, operands[0], operands[1],
+			   NULL_RTX, NULL_RTX, operands[2]);
+  DONE;
+})
+
+(define_expand "atomic_fetch_<fetchop_name><mode>"
+  [(match_operand:AINT 0 "int_reg_operand" "")		;; output
+   (match_operand:AINT 1 "memory_operand" "")		;; memory
+   (FETCHOP:AINT (match_dup 1)
+     (match_operand:AINT 2 "<fetchop_pred>" ""))	;; operand
+   (match_operand:SI 3 "const_int_operand" "")]		;; model
+  ""
+{ 
+  rs6000_expand_atomic_op (<CODE>, operands[1], operands[2],
+			   operands[0], NULL_RTX, operands[3]);
+  DONE;
+})
 
+(define_expand "atomic_fetch_nand<mode>"
+  [(match_operand:AINT 0 "int_reg_operand" "")		;; output
+   (match_operand:AINT 1 "memory_operand" "")		;; memory
+   (match_operand:AINT 2 "int_reg_operand" "")		;; operand
+   (match_operand:SI 3 "const_int_operand" "")]		;; model
+  ""
+{
+  rs6000_expand_atomic_op (NOT, operands[1], operands[2],
+			   operands[0], NULL_RTX, operands[3]);
+  DONE;
+})
+
+(define_expand "atomic_<fetchop_name>_fetch<mode>"
+  [(match_operand:AINT 0 "int_reg_operand" "")		;; output
+   (match_operand:AINT 1 "memory_operand" "")		;; memory
+   (FETCHOP:AINT (match_dup 1)
+     (match_operand:AINT 2 "<fetchop_pred>" ""))	;; operand
+   (match_operand:SI 3 "const_int_operand" "")]		;; model
+  ""
+{
+  rs6000_expand_atomic_op (<CODE>, operands[1], operands[2],
+			   NULL_RTX, operands[0], operands[3]);
+  DONE;
+})
+
+(define_expand "atomic_nand_fetch<mode>"
+  [(match_operand:AINT 0 "int_reg_operand" "")		;; output
+   (match_operand:AINT 1 "memory_operand" "")		;; memory
+   (match_operand:AINT 2 "int_reg_operand" "")		;; operand
+   (match_operand:SI 3 "const_int_operand" "")]		;; model
+  ""
+{
+  rs6000_expand_atomic_op (NOT, operands[1], operands[2],
+			   NULL_RTX, operands[0], operands[3]);
+  DONE;
+})