cvs.zerfleddert.de Git - proxmark3-svn/blame_incremental

... / ...

Commit	Line	Data
	1	/**
	2	* \file bn_mul.h
	3	*
	4	* \brief Multi-precision integer library
	5	*/
	6	/*
	7	* Copyright (C) 2006-2015, ARM Limited, All Rights Reserved
	8	* SPDX-License-Identifier: GPL-2.0
	9	*
	10	* This program is free software; you can redistribute it and/or modify
	11	* it under the terms of the GNU General Public License as published by
	12	* the Free Software Foundation; either version 2 of the License, or
	13	* (at your option) any later version.
	14	*
	15	* This program is distributed in the hope that it will be useful,
	16	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	18	* GNU General Public License for more details.
	19	*
	20	* You should have received a copy of the GNU General Public License along
	21	* with this program; if not, write to the Free Software Foundation, Inc.,
	22	* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	23	*
	24	* This file is part of mbed TLS (https://tls.mbed.org)
	25	*/
	26	/*
	27	* Multiply source vector [s] with b, add result
	28	* to destination vector [d] and set carry c.
	29	*
	30	* Currently supports:
	31	*
	32	* . IA-32 (386+) . AMD64 / EM64T
	33	* . IA-32 (SSE2) . Motorola 68000
	34	* . PowerPC, 32-bit . MicroBlaze
	35	* . PowerPC, 64-bit . TriCore
	36	* . SPARC v8 . ARM v3+
	37	* . Alpha . MIPS32
	38	* . C, longlong . C, generic
	39	*/
	40	#ifndef MBEDTLS_BN_MUL_H
	41	#define MBEDTLS_BN_MUL_H
	42
	43	#include "bignum.h"
	44
	45	#if defined(MBEDTLS_HAVE_ASM)
	46
	47	#ifndef asm
	48	#define asm __asm
	49	#endif
	50
	51	/* armcc5 --gnu defines __GNUC__ but doesn't support GNU's extended asm */
	52	#if defined(__GNUC__) && \
	53	( !defined(__ARMCC_VERSION) \|\| __ARMCC_VERSION >= 6000000 )
	54
	55	/*
	56	* Disable use of the i386 assembly code below if option -O0, to disable all
	57	* compiler optimisations, is passed, detected with __OPTIMIZE__
	58	* This is done as the number of registers used in the assembly code doesn't
	59	* work with the -O0 option.
	60	*/
	61	#if defined(__i386__) && defined(__OPTIMIZE__)
	62
	63	#define MULADDC_INIT \
	64	asm( \
	65	"movl %%ebx, %0 \n\t" \
	66	"movl %5, %%esi \n\t" \
	67	"movl %6, %%edi \n\t" \
	68	"movl %7, %%ecx \n\t" \
	69	"movl %8, %%ebx \n\t"
	70
	71	#define MULADDC_CORE \
	72	"lodsl \n\t" \
	73	"mull %%ebx \n\t" \
	74	"addl %%ecx, %%eax \n\t" \
	75	"adcl $0, %%edx \n\t" \
	76	"addl (%%edi), %%eax \n\t" \
	77	"adcl $0, %%edx \n\t" \
	78	"movl %%edx, %%ecx \n\t" \
	79	"stosl \n\t"
	80
	81	#if defined(MBEDTLS_HAVE_SSE2)
	82
	83	#define MULADDC_HUIT \
	84	"movd %%ecx, %%mm1 \n\t" \
	85	"movd %%ebx, %%mm0 \n\t" \
	86	"movd (%%edi), %%mm3 \n\t" \
	87	"paddq %%mm3, %%mm1 \n\t" \
	88	"movd (%%esi), %%mm2 \n\t" \
	89	"pmuludq %%mm0, %%mm2 \n\t" \
	90	"movd 4(%%esi), %%mm4 \n\t" \
	91	"pmuludq %%mm0, %%mm4 \n\t" \
	92	"movd 8(%%esi), %%mm6 \n\t" \
	93	"pmuludq %%mm0, %%mm6 \n\t" \
	94	"movd 12(%%esi), %%mm7 \n\t" \
	95	"pmuludq %%mm0, %%mm7 \n\t" \
	96	"paddq %%mm2, %%mm1 \n\t" \
	97	"movd 4(%%edi), %%mm3 \n\t" \
	98	"paddq %%mm4, %%mm3 \n\t" \
	99	"movd 8(%%edi), %%mm5 \n\t" \
	100	"paddq %%mm6, %%mm5 \n\t" \
	101	"movd 12(%%edi), %%mm4 \n\t" \
	102	"paddq %%mm4, %%mm7 \n\t" \
	103	"movd %%mm1, (%%edi) \n\t" \
	104	"movd 16(%%esi), %%mm2 \n\t" \
	105	"pmuludq %%mm0, %%mm2 \n\t" \
	106	"psrlq $32, %%mm1 \n\t" \
	107	"movd 20(%%esi), %%mm4 \n\t" \
	108	"pmuludq %%mm0, %%mm4 \n\t" \
	109	"paddq %%mm3, %%mm1 \n\t" \
	110	"movd 24(%%esi), %%mm6 \n\t" \
	111	"pmuludq %%mm0, %%mm6 \n\t" \
	112	"movd %%mm1, 4(%%edi) \n\t" \
	113	"psrlq $32, %%mm1 \n\t" \
	114	"movd 28(%%esi), %%mm3 \n\t" \
	115	"pmuludq %%mm0, %%mm3 \n\t" \
	116	"paddq %%mm5, %%mm1 \n\t" \
	117	"movd 16(%%edi), %%mm5 \n\t" \
	118	"paddq %%mm5, %%mm2 \n\t" \
	119	"movd %%mm1, 8(%%edi) \n\t" \
	120	"psrlq $32, %%mm1 \n\t" \
	121	"paddq %%mm7, %%mm1 \n\t" \
	122	"movd 20(%%edi), %%mm5 \n\t" \
	123	"paddq %%mm5, %%mm4 \n\t" \
	124	"movd %%mm1, 12(%%edi) \n\t" \
	125	"psrlq $32, %%mm1 \n\t" \
	126	"paddq %%mm2, %%mm1 \n\t" \
	127	"movd 24(%%edi), %%mm5 \n\t" \
	128	"paddq %%mm5, %%mm6 \n\t" \
	129	"movd %%mm1, 16(%%edi) \n\t" \
	130	"psrlq $32, %%mm1 \n\t" \
	131	"paddq %%mm4, %%mm1 \n\t" \
	132	"movd 28(%%edi), %%mm5 \n\t" \
	133	"paddq %%mm5, %%mm3 \n\t" \
	134	"movd %%mm1, 20(%%edi) \n\t" \
	135	"psrlq $32, %%mm1 \n\t" \
	136	"paddq %%mm6, %%mm1 \n\t" \
	137	"movd %%mm1, 24(%%edi) \n\t" \
	138	"psrlq $32, %%mm1 \n\t" \
	139	"paddq %%mm3, %%mm1 \n\t" \
	140	"movd %%mm1, 28(%%edi) \n\t" \
	141	"addl $32, %%edi \n\t" \
	142	"addl $32, %%esi \n\t" \
	143	"psrlq $32, %%mm1 \n\t" \
	144	"movd %%mm1, %%ecx \n\t"
	145
	146	#define MULADDC_STOP \
	147	"emms \n\t" \
	148	"movl %4, %%ebx \n\t" \
	149	"movl %%ecx, %1 \n\t" \
	150	"movl %%edi, %2 \n\t" \
	151	"movl %%esi, %3 \n\t" \
	152	: "=m" (t), "=m" (c), "=m" (d), "=m" (s) \
	153	: "m" (t), "m" (s), "m" (d), "m" (c), "m" (b) \
	154	: "eax", "ebx", "ecx", "edx", "esi", "edi" \
	155	);
	156
	157	#else
	158
	159	#define MULADDC_STOP \
	160	"movl %4, %%ebx \n\t" \
	161	"movl %%ecx, %1 \n\t" \
	162	"movl %%edi, %2 \n\t" \
	163	"movl %%esi, %3 \n\t" \
	164	: "=m" (t), "=m" (c), "=m" (d), "=m" (s) \
	165	: "m" (t), "m" (s), "m" (d), "m" (c), "m" (b) \
	166	: "eax", "ebx", "ecx", "edx", "esi", "edi" \
	167	);
	168	#endif /* SSE2 */
	169	#endif /* i386 */
	170
	171	#if defined(__amd64__) \|\| defined (__x86_64__)
	172
	173	#define MULADDC_INIT \
	174	asm( \
	175	"xorq %%r8, %%r8 \n\t"
	176
	177	#define MULADDC_CORE \
	178	"movq (%%rsi), %%rax \n\t" \
	179	"mulq %%rbx \n\t" \
	180	"addq $8, %%rsi \n\t" \
	181	"addq %%rcx, %%rax \n\t" \
	182	"movq %%r8, %%rcx \n\t" \
	183	"adcq $0, %%rdx \n\t" \
	184	"nop \n\t" \
	185	"addq %%rax, (%%rdi) \n\t" \
	186	"adcq %%rdx, %%rcx \n\t" \
	187	"addq $8, %%rdi \n\t"
	188
	189	#define MULADDC_STOP \
	190	: "+c" (c), "+D" (d), "+S" (s) \
	191	: "b" (b) \
	192	: "rax", "rdx", "r8" \
	193	);
	194
	195	#endif /* AMD64 */
	196
	197	#if defined(__mc68020__) \|\| defined(__mcpu32__)
	198
	199	#define MULADDC_INIT \
	200	asm( \
	201	"movl %3, %%a2 \n\t" \
	202	"movl %4, %%a3 \n\t" \
	203	"movl %5, %%d3 \n\t" \
	204	"movl %6, %%d2 \n\t" \
	205	"moveq #0, %%d0 \n\t"
	206
	207	#define MULADDC_CORE \
	208	"movel %%a2@+, %%d1 \n\t" \
	209	"mulul %%d2, %%d4:%%d1 \n\t" \
	210	"addl %%d3, %%d1 \n\t" \
	211	"addxl %%d0, %%d4 \n\t" \
	212	"moveq #0, %%d3 \n\t" \
	213	"addl %%d1, %%a3@+ \n\t" \
	214	"addxl %%d4, %%d3 \n\t"
	215
	216	#define MULADDC_STOP \
	217	"movl %%d3, %0 \n\t" \
	218	"movl %%a3, %1 \n\t" \
	219	"movl %%a2, %2 \n\t" \
	220	: "=m" (c), "=m" (d), "=m" (s) \
	221	: "m" (s), "m" (d), "m" (c), "m" (b) \
	222	: "d0", "d1", "d2", "d3", "d4", "a2", "a3" \
	223	);
	224
	225	#define MULADDC_HUIT \
	226	"movel %%a2@+, %%d1 \n\t" \
	227	"mulul %%d2, %%d4:%%d1 \n\t" \
	228	"addxl %%d3, %%d1 \n\t" \
	229	"addxl %%d0, %%d4 \n\t" \
	230	"addl %%d1, %%a3@+ \n\t" \
	231	"movel %%a2@+, %%d1 \n\t" \
	232	"mulul %%d2, %%d3:%%d1 \n\t" \
	233	"addxl %%d4, %%d1 \n\t" \
	234	"addxl %%d0, %%d3 \n\t" \
	235	"addl %%d1, %%a3@+ \n\t" \
	236	"movel %%a2@+, %%d1 \n\t" \
	237	"mulul %%d2, %%d4:%%d1 \n\t" \
	238	"addxl %%d3, %%d1 \n\t" \
	239	"addxl %%d0, %%d4 \n\t" \
	240	"addl %%d1, %%a3@+ \n\t" \
	241	"movel %%a2@+, %%d1 \n\t" \
	242	"mulul %%d2, %%d3:%%d1 \n\t" \
	243	"addxl %%d4, %%d1 \n\t" \
	244	"addxl %%d0, %%d3 \n\t" \
	245	"addl %%d1, %%a3@+ \n\t" \
	246	"movel %%a2@+, %%d1 \n\t" \
	247	"mulul %%d2, %%d4:%%d1 \n\t" \
	248	"addxl %%d3, %%d1 \n\t" \
	249	"addxl %%d0, %%d4 \n\t" \
	250	"addl %%d1, %%a3@+ \n\t" \
	251	"movel %%a2@+, %%d1 \n\t" \
	252	"mulul %%d2, %%d3:%%d1 \n\t" \
	253	"addxl %%d4, %%d1 \n\t" \
	254	"addxl %%d0, %%d3 \n\t" \
	255	"addl %%d1, %%a3@+ \n\t" \
	256	"movel %%a2@+, %%d1 \n\t" \
	257	"mulul %%d2, %%d4:%%d1 \n\t" \
	258	"addxl %%d3, %%d1 \n\t" \
	259	"addxl %%d0, %%d4 \n\t" \
	260	"addl %%d1, %%a3@+ \n\t" \
	261	"movel %%a2@+, %%d1 \n\t" \
	262	"mulul %%d2, %%d3:%%d1 \n\t" \
	263	"addxl %%d4, %%d1 \n\t" \
	264	"addxl %%d0, %%d3 \n\t" \
	265	"addl %%d1, %%a3@+ \n\t" \
	266	"addxl %%d0, %%d3 \n\t"
	267
	268	#endif /* MC68000 */
	269
	270	#if defined(__powerpc64__) \|\| defined(__ppc64__)
	271
	272	#if defined(__MACH__) && defined(__APPLE__)
	273
	274	#define MULADDC_INIT \
	275	asm( \
	276	"ld r3, %3 \n\t" \
	277	"ld r4, %4 \n\t" \
	278	"ld r5, %5 \n\t" \
	279	"ld r6, %6 \n\t" \
	280	"addi r3, r3, -8 \n\t" \
	281	"addi r4, r4, -8 \n\t" \
	282	"addic r5, r5, 0 \n\t"
	283
	284	#define MULADDC_CORE \
	285	"ldu r7, 8(r3) \n\t" \
	286	"mulld r8, r7, r6 \n\t" \
	287	"mulhdu r9, r7, r6 \n\t" \
	288	"adde r8, r8, r5 \n\t" \
	289	"ld r7, 8(r4) \n\t" \
	290	"addze r5, r9 \n\t" \
	291	"addc r8, r8, r7 \n\t" \
	292	"stdu r8, 8(r4) \n\t"
	293
	294	#define MULADDC_STOP \
	295	"addze r5, r5 \n\t" \
	296	"addi r4, r4, 8 \n\t" \
	297	"addi r3, r3, 8 \n\t" \
	298	"std r5, %0 \n\t" \
	299	"std r4, %1 \n\t" \
	300	"std r3, %2 \n\t" \
	301	: "=m" (c), "=m" (d), "=m" (s) \
	302	: "m" (s), "m" (d), "m" (c), "m" (b) \
	303	: "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
	304	);
	305
	306
	307	#else /* __MACH__ && __APPLE__ */
	308
	309	#define MULADDC_INIT \
	310	asm( \
	311	"ld %%r3, %3 \n\t" \
	312	"ld %%r4, %4 \n\t" \
	313	"ld %%r5, %5 \n\t" \
	314	"ld %%r6, %6 \n\t" \
	315	"addi %%r3, %%r3, -8 \n\t" \
	316	"addi %%r4, %%r4, -8 \n\t" \
	317	"addic %%r5, %%r5, 0 \n\t"
	318
	319	#define MULADDC_CORE \
	320	"ldu %%r7, 8(%%r3) \n\t" \
	321	"mulld %%r8, %%r7, %%r6 \n\t" \
	322	"mulhdu %%r9, %%r7, %%r6 \n\t" \
	323	"adde %%r8, %%r8, %%r5 \n\t" \
	324	"ld %%r7, 8(%%r4) \n\t" \
	325	"addze %%r5, %%r9 \n\t" \
	326	"addc %%r8, %%r8, %%r7 \n\t" \
	327	"stdu %%r8, 8(%%r4) \n\t"
	328
	329	#define MULADDC_STOP \
	330	"addze %%r5, %%r5 \n\t" \
	331	"addi %%r4, %%r4, 8 \n\t" \
	332	"addi %%r3, %%r3, 8 \n\t" \
	333	"std %%r5, %0 \n\t" \
	334	"std %%r4, %1 \n\t" \
	335	"std %%r3, %2 \n\t" \
	336	: "=m" (c), "=m" (d), "=m" (s) \
	337	: "m" (s), "m" (d), "m" (c), "m" (b) \
	338	: "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
	339	);
	340
	341	#endif /* __MACH__ && __APPLE__ */
	342
	343	#elif defined(__powerpc__) \|\| defined(__ppc__) /* end PPC64/begin PPC32 */
	344
	345	#if defined(__MACH__) && defined(__APPLE__)
	346
	347	#define MULADDC_INIT \
	348	asm( \
	349	"lwz r3, %3 \n\t" \
	350	"lwz r4, %4 \n\t" \
	351	"lwz r5, %5 \n\t" \
	352	"lwz r6, %6 \n\t" \
	353	"addi r3, r3, -4 \n\t" \
	354	"addi r4, r4, -4 \n\t" \
	355	"addic r5, r5, 0 \n\t"
	356
	357	#define MULADDC_CORE \
	358	"lwzu r7, 4(r3) \n\t" \
	359	"mullw r8, r7, r6 \n\t" \
	360	"mulhwu r9, r7, r6 \n\t" \
	361	"adde r8, r8, r5 \n\t" \
	362	"lwz r7, 4(r4) \n\t" \
	363	"addze r5, r9 \n\t" \
	364	"addc r8, r8, r7 \n\t" \
	365	"stwu r8, 4(r4) \n\t"
	366
	367	#define MULADDC_STOP \
	368	"addze r5, r5 \n\t" \
	369	"addi r4, r4, 4 \n\t" \
	370	"addi r3, r3, 4 \n\t" \
	371	"stw r5, %0 \n\t" \
	372	"stw r4, %1 \n\t" \
	373	"stw r3, %2 \n\t" \
	374	: "=m" (c), "=m" (d), "=m" (s) \
	375	: "m" (s), "m" (d), "m" (c), "m" (b) \
	376	: "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
	377	);
	378
	379	#else /* __MACH__ && __APPLE__ */
	380
	381	#define MULADDC_INIT \
	382	asm( \
	383	"lwz %%r3, %3 \n\t" \
	384	"lwz %%r4, %4 \n\t" \
	385	"lwz %%r5, %5 \n\t" \
	386	"lwz %%r6, %6 \n\t" \
	387	"addi %%r3, %%r3, -4 \n\t" \
	388	"addi %%r4, %%r4, -4 \n\t" \
	389	"addic %%r5, %%r5, 0 \n\t"
	390
	391	#define MULADDC_CORE \
	392	"lwzu %%r7, 4(%%r3) \n\t" \
	393	"mullw %%r8, %%r7, %%r6 \n\t" \
	394	"mulhwu %%r9, %%r7, %%r6 \n\t" \
	395	"adde %%r8, %%r8, %%r5 \n\t" \
	396	"lwz %%r7, 4(%%r4) \n\t" \
	397	"addze %%r5, %%r9 \n\t" \
	398	"addc %%r8, %%r8, %%r7 \n\t" \
	399	"stwu %%r8, 4(%%r4) \n\t"
	400
	401	#define MULADDC_STOP \
	402	"addze %%r5, %%r5 \n\t" \
	403	"addi %%r4, %%r4, 4 \n\t" \
	404	"addi %%r3, %%r3, 4 \n\t" \
	405	"stw %%r5, %0 \n\t" \
	406	"stw %%r4, %1 \n\t" \
	407	"stw %%r3, %2 \n\t" \
	408	: "=m" (c), "=m" (d), "=m" (s) \
	409	: "m" (s), "m" (d), "m" (c), "m" (b) \
	410	: "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
	411	);
	412
	413	#endif /* __MACH__ && __APPLE__ */
	414
	415	#endif /* PPC32 */
	416
	417	/*
	418	* The Sparc(64) assembly is reported to be broken.
	419	* Disable it for now, until we're able to fix it.
	420	*/
	421	#if 0 && defined(__sparc__)
	422	#if defined(__sparc64__)
	423
	424	#define MULADDC_INIT \
	425	asm( \
	426	"ldx %3, %%o0 \n\t" \
	427	"ldx %4, %%o1 \n\t" \
	428	"ld %5, %%o2 \n\t" \
	429	"ld %6, %%o3 \n\t"
	430
	431	#define MULADDC_CORE \
	432	"ld [%%o0], %%o4 \n\t" \
	433	"inc 4, %%o0 \n\t" \
	434	"ld [%%o1], %%o5 \n\t" \
	435	"umul %%o3, %%o4, %%o4 \n\t" \
	436	"addcc %%o4, %%o2, %%o4 \n\t" \
	437	"rd %%y, %%g1 \n\t" \
	438	"addx %%g1, 0, %%g1 \n\t" \
	439	"addcc %%o4, %%o5, %%o4 \n\t" \
	440	"st %%o4, [%%o1] \n\t" \
	441	"addx %%g1, 0, %%o2 \n\t" \
	442	"inc 4, %%o1 \n\t"
	443
	444	#define MULADDC_STOP \
	445	"st %%o2, %0 \n\t" \
	446	"stx %%o1, %1 \n\t" \
	447	"stx %%o0, %2 \n\t" \
	448	: "=m" (c), "=m" (d), "=m" (s) \
	449	: "m" (s), "m" (d), "m" (c), "m" (b) \
	450	: "g1", "o0", "o1", "o2", "o3", "o4", \
	451	"o5" \
	452	);
	453
	454	#else /* __sparc64__ */
	455
	456	#define MULADDC_INIT \
	457	asm( \
	458	"ld %3, %%o0 \n\t" \
	459	"ld %4, %%o1 \n\t" \
	460	"ld %5, %%o2 \n\t" \
	461	"ld %6, %%o3 \n\t"
	462
	463	#define MULADDC_CORE \
	464	"ld [%%o0], %%o4 \n\t" \
	465	"inc 4, %%o0 \n\t" \
	466	"ld [%%o1], %%o5 \n\t" \
	467	"umul %%o3, %%o4, %%o4 \n\t" \
	468	"addcc %%o4, %%o2, %%o4 \n\t" \
	469	"rd %%y, %%g1 \n\t" \
	470	"addx %%g1, 0, %%g1 \n\t" \
	471	"addcc %%o4, %%o5, %%o4 \n\t" \
	472	"st %%o4, [%%o1] \n\t" \
	473	"addx %%g1, 0, %%o2 \n\t" \
	474	"inc 4, %%o1 \n\t"
	475
	476	#define MULADDC_STOP \
	477	"st %%o2, %0 \n\t" \
	478	"st %%o1, %1 \n\t" \
	479	"st %%o0, %2 \n\t" \
	480	: "=m" (c), "=m" (d), "=m" (s) \
	481	: "m" (s), "m" (d), "m" (c), "m" (b) \
	482	: "g1", "o0", "o1", "o2", "o3", "o4", \
	483	"o5" \
	484	);
	485
	486	#endif /* __sparc64__ */
	487	#endif /* __sparc__ */
	488
	489	#if defined(__microblaze__) \|\| defined(microblaze)
	490
	491	#define MULADDC_INIT \
	492	asm( \
	493	"lwi r3, %3 \n\t" \
	494	"lwi r4, %4 \n\t" \
	495	"lwi r5, %5 \n\t" \
	496	"lwi r6, %6 \n\t" \
	497	"andi r7, r6, 0xffff \n\t" \
	498	"bsrli r6, r6, 16 \n\t"
	499
	500	#define MULADDC_CORE \
	501	"lhui r8, r3, 0 \n\t" \
	502	"addi r3, r3, 2 \n\t" \
	503	"lhui r9, r3, 0 \n\t" \
	504	"addi r3, r3, 2 \n\t" \
	505	"mul r10, r9, r6 \n\t" \
	506	"mul r11, r8, r7 \n\t" \
	507	"mul r12, r9, r7 \n\t" \
	508	"mul r13, r8, r6 \n\t" \
	509	"bsrli r8, r10, 16 \n\t" \
	510	"bsrli r9, r11, 16 \n\t" \
	511	"add r13, r13, r8 \n\t" \
	512	"add r13, r13, r9 \n\t" \
	513	"bslli r10, r10, 16 \n\t" \
	514	"bslli r11, r11, 16 \n\t" \
	515	"add r12, r12, r10 \n\t" \
	516	"addc r13, r13, r0 \n\t" \
	517	"add r12, r12, r11 \n\t" \
	518	"addc r13, r13, r0 \n\t" \
	519	"lwi r10, r4, 0 \n\t" \
	520	"add r12, r12, r10 \n\t" \
	521	"addc r13, r13, r0 \n\t" \
	522	"add r12, r12, r5 \n\t" \
	523	"addc r5, r13, r0 \n\t" \
	524	"swi r12, r4, 0 \n\t" \
	525	"addi r4, r4, 4 \n\t"
	526
	527	#define MULADDC_STOP \
	528	"swi r5, %0 \n\t" \
	529	"swi r4, %1 \n\t" \
	530	"swi r3, %2 \n\t" \
	531	: "=m" (c), "=m" (d), "=m" (s) \
	532	: "m" (s), "m" (d), "m" (c), "m" (b) \
	533	: "r3", "r4", "r5", "r6", "r7", "r8", \
	534	"r9", "r10", "r11", "r12", "r13" \
	535	);
	536
	537	#endif /* MicroBlaze */
	538
	539	#if defined(__tricore__)
	540
	541	#define MULADDC_INIT \
	542	asm( \
	543	"ld.a %%a2, %3 \n\t" \
	544	"ld.a %%a3, %4 \n\t" \
	545	"ld.w %%d4, %5 \n\t" \
	546	"ld.w %%d1, %6 \n\t" \
	547	"xor %%d5, %%d5 \n\t"
	548
	549	#define MULADDC_CORE \
	550	"ld.w %%d0, [%%a2+] \n\t" \
	551	"madd.u %%e2, %%e4, %%d0, %%d1 \n\t" \
	552	"ld.w %%d0, [%%a3] \n\t" \
	553	"addx %%d2, %%d2, %%d0 \n\t" \
	554	"addc %%d3, %%d3, 0 \n\t" \
	555	"mov %%d4, %%d3 \n\t" \
	556	"st.w [%%a3+], %%d2 \n\t"
	557
	558	#define MULADDC_STOP \
	559	"st.w %0, %%d4 \n\t" \
	560	"st.a %1, %%a3 \n\t" \
	561	"st.a %2, %%a2 \n\t" \
	562	: "=m" (c), "=m" (d), "=m" (s) \
	563	: "m" (s), "m" (d), "m" (c), "m" (b) \
	564	: "d0", "d1", "e2", "d4", "a2", "a3" \
	565	);
	566
	567	#endif /* TriCore */
	568
	569	/*
	570	* gcc -O0 by default uses r7 for the frame pointer, so it complains about our
	571	* use of r7 below, unless -fomit-frame-pointer is passed. Unfortunately,
	572	* passing that option is not easy when building with yotta.
	573	*
	574	* On the other hand, -fomit-frame-pointer is implied by any -Ox options with
	575	* x !=0, which we can detect using __OPTIMIZE__ (which is also defined by
	576	* clang and armcc5 under the same conditions).
	577	*
	578	* So, only use the optimized assembly below for optimized build, which avoids
	579	* the build error and is pretty reasonable anyway.
	580	*/
	581	#if defined(__GNUC__) && !defined(__OPTIMIZE__)
	582	#define MULADDC_CANNOT_USE_R7
	583	#endif
	584
	585	#if defined(__arm__) && !defined(MULADDC_CANNOT_USE_R7)
	586
	587	#if defined(__thumb__) && !defined(__thumb2__)
	588
	589	#define MULADDC_INIT \
	590	asm( \
	591	"ldr r0, %3 \n\t" \
	592	"ldr r1, %4 \n\t" \
	593	"ldr r2, %5 \n\t" \
	594	"ldr r3, %6 \n\t" \
	595	"lsr r7, r3, #16 \n\t" \
	596	"mov r9, r7 \n\t" \
	597	"lsl r7, r3, #16 \n\t" \
	598	"lsr r7, r7, #16 \n\t" \
	599	"mov r8, r7 \n\t"
	600
	601	#define MULADDC_CORE \
	602	"ldmia r0!, {r6} \n\t" \
	603	"lsr r7, r6, #16 \n\t" \
	604	"lsl r6, r6, #16 \n\t" \
	605	"lsr r6, r6, #16 \n\t" \
	606	"mov r4, r8 \n\t" \
	607	"mul r4, r6 \n\t" \
	608	"mov r3, r9 \n\t" \
	609	"mul r6, r3 \n\t" \
	610	"mov r5, r9 \n\t" \
	611	"mul r5, r7 \n\t" \
	612	"mov r3, r8 \n\t" \
	613	"mul r7, r3 \n\t" \
	614	"lsr r3, r6, #16 \n\t" \
	615	"add r5, r5, r3 \n\t" \
	616	"lsr r3, r7, #16 \n\t" \
	617	"add r5, r5, r3 \n\t" \
	618	"add r4, r4, r2 \n\t" \
	619	"mov r2, #0 \n\t" \
	620	"adc r5, r2 \n\t" \
	621	"lsl r3, r6, #16 \n\t" \
	622	"add r4, r4, r3 \n\t" \
	623	"adc r5, r2 \n\t" \
	624	"lsl r3, r7, #16 \n\t" \
	625	"add r4, r4, r3 \n\t" \
	626	"adc r5, r2 \n\t" \
	627	"ldr r3, [r1] \n\t" \
	628	"add r4, r4, r3 \n\t" \
	629	"adc r2, r5 \n\t" \
	630	"stmia r1!, {r4} \n\t"
	631
	632	#define MULADDC_STOP \
	633	"str r2, %0 \n\t" \
	634	"str r1, %1 \n\t" \
	635	"str r0, %2 \n\t" \
	636	: "=m" (c), "=m" (d), "=m" (s) \
	637	: "m" (s), "m" (d), "m" (c), "m" (b) \
	638	: "r0", "r1", "r2", "r3", "r4", "r5", \
	639	"r6", "r7", "r8", "r9", "cc" \
	640	);
	641
	642	#else
	643
	644	#define MULADDC_INIT \
	645	asm( \
	646	"ldr r0, %3 \n\t" \
	647	"ldr r1, %4 \n\t" \
	648	"ldr r2, %5 \n\t" \
	649	"ldr r3, %6 \n\t"
	650
	651	#define MULADDC_CORE \
	652	"ldr r4, [r0], #4 \n\t" \
	653	"mov r5, #0 \n\t" \
	654	"ldr r6, [r1] \n\t" \
	655	"umlal r2, r5, r3, r4 \n\t" \
	656	"adds r7, r6, r2 \n\t" \
	657	"adc r2, r5, #0 \n\t" \
	658	"str r7, [r1], #4 \n\t"
	659
	660	#define MULADDC_STOP \
	661	"str r2, %0 \n\t" \
	662	"str r1, %1 \n\t" \
	663	"str r0, %2 \n\t" \
	664	: "=m" (c), "=m" (d), "=m" (s) \
	665	: "m" (s), "m" (d), "m" (c), "m" (b) \
	666	: "r0", "r1", "r2", "r3", "r4", "r5", \
	667	"r6", "r7", "cc" \
	668	);
	669
	670	#endif /* Thumb */
	671
	672	#endif /* ARMv3 */
	673
	674	#if defined(__alpha__)
	675
	676	#define MULADDC_INIT \
	677	asm( \
	678	"ldq $1, %3 \n\t" \
	679	"ldq $2, %4 \n\t" \
	680	"ldq $3, %5 \n\t" \
	681	"ldq $4, %6 \n\t"
	682
	683	#define MULADDC_CORE \
	684	"ldq $6, 0($1) \n\t" \
	685	"addq $1, 8, $1 \n\t" \
	686	"mulq $6, $4, $7 \n\t" \
	687	"umulh $6, $4, $6 \n\t" \
	688	"addq $7, $3, $7 \n\t" \
	689	"cmpult $7, $3, $3 \n\t" \
	690	"ldq $5, 0($2) \n\t" \
	691	"addq $7, $5, $7 \n\t" \
	692	"cmpult $7, $5, $5 \n\t" \
	693	"stq $7, 0($2) \n\t" \
	694	"addq $2, 8, $2 \n\t" \
	695	"addq $6, $3, $3 \n\t" \
	696	"addq $5, $3, $3 \n\t"
	697
	698	#define MULADDC_STOP \
	699	"stq $3, %0 \n\t" \
	700	"stq $2, %1 \n\t" \
	701	"stq $1, %2 \n\t" \
	702	: "=m" (c), "=m" (d), "=m" (s) \
	703	: "m" (s), "m" (d), "m" (c), "m" (b) \
	704	: "$1", "$2", "$3", "$4", "$5", "$6", "$7" \
	705	);
	706	#endif /* Alpha */
	707
	708	#if defined(__mips__) && !defined(__mips64)
	709
	710	#define MULADDC_INIT \
	711	asm( \
	712	"lw $10, %3 \n\t" \
	713	"lw $11, %4 \n\t" \
	714	"lw $12, %5 \n\t" \
	715	"lw $13, %6 \n\t"
	716
	717	#define MULADDC_CORE \
	718	"lw $14, 0($10) \n\t" \
	719	"multu $13, $14 \n\t" \
	720	"addi $10, $10, 4 \n\t" \
	721	"mflo $14 \n\t" \
	722	"mfhi $9 \n\t" \
	723	"addu $14, $12, $14 \n\t" \
	724	"lw $15, 0($11) \n\t" \
	725	"sltu $12, $14, $12 \n\t" \
	726	"addu $15, $14, $15 \n\t" \
	727	"sltu $14, $15, $14 \n\t" \
	728	"addu $12, $12, $9 \n\t" \
	729	"sw $15, 0($11) \n\t" \
	730	"addu $12, $12, $14 \n\t" \
	731	"addi $11, $11, 4 \n\t"
	732
	733	#define MULADDC_STOP \
	734	"sw $12, %0 \n\t" \
	735	"sw $11, %1 \n\t" \
	736	"sw $10, %2 \n\t" \
	737	: "=m" (c), "=m" (d), "=m" (s) \
	738	: "m" (s), "m" (d), "m" (c), "m" (b) \
	739	: "$9", "$10", "$11", "$12", "$13", "$14", "$15" \
	740	);
	741
	742	#endif /* MIPS */
	743	#endif /* GNUC */
	744
	745	#if (defined(_MSC_VER) && defined(_M_IX86)) \|\| defined(__WATCOMC__)
	746
	747	#define MULADDC_INIT \
	748	__asm mov esi, s \
	749	__asm mov edi, d \
	750	__asm mov ecx, c \
	751	__asm mov ebx, b
	752
	753	#define MULADDC_CORE \
	754	__asm lodsd \
	755	__asm mul ebx \
	756	__asm add eax, ecx \
	757	__asm adc edx, 0 \
	758	__asm add eax, [edi] \
	759	__asm adc edx, 0 \
	760	__asm mov ecx, edx \
	761	__asm stosd
	762
	763	#if defined(MBEDTLS_HAVE_SSE2)
	764
	765	#define EMIT __asm _emit
	766
	767	#define MULADDC_HUIT \
	768	EMIT 0x0F EMIT 0x6E EMIT 0xC9 \
	769	EMIT 0x0F EMIT 0x6E EMIT 0xC3 \
	770	EMIT 0x0F EMIT 0x6E EMIT 0x1F \
	771	EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
	772	EMIT 0x0F EMIT 0x6E EMIT 0x16 \
	773	EMIT 0x0F EMIT 0xF4 EMIT 0xD0 \
	774	EMIT 0x0F EMIT 0x6E EMIT 0x66 EMIT 0x04 \
	775	EMIT 0x0F EMIT 0xF4 EMIT 0xE0 \
	776	EMIT 0x0F EMIT 0x6E EMIT 0x76 EMIT 0x08 \
	777	EMIT 0x0F EMIT 0xF4 EMIT 0xF0 \
	778	EMIT 0x0F EMIT 0x6E EMIT 0x7E EMIT 0x0C \
	779	EMIT 0x0F EMIT 0xF4 EMIT 0xF8 \
	780	EMIT 0x0F EMIT 0xD4 EMIT 0xCA \
	781	EMIT 0x0F EMIT 0x6E EMIT 0x5F EMIT 0x04 \
	782	EMIT 0x0F EMIT 0xD4 EMIT 0xDC \
	783	EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x08 \
	784	EMIT 0x0F EMIT 0xD4 EMIT 0xEE \
	785	EMIT 0x0F EMIT 0x6E EMIT 0x67 EMIT 0x0C \
	786	EMIT 0x0F EMIT 0xD4 EMIT 0xFC \
	787	EMIT 0x0F EMIT 0x7E EMIT 0x0F \
	788	EMIT 0x0F EMIT 0x6E EMIT 0x56 EMIT 0x10 \
	789	EMIT 0x0F EMIT 0xF4 EMIT 0xD0 \
	790	EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
	791	EMIT 0x0F EMIT 0x6E EMIT 0x66 EMIT 0x14 \
	792	EMIT 0x0F EMIT 0xF4 EMIT 0xE0 \
	793	EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
	794	EMIT 0x0F EMIT 0x6E EMIT 0x76 EMIT 0x18 \
	795	EMIT 0x0F EMIT 0xF4 EMIT 0xF0 \
	796	EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x04 \
	797	EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
	798	EMIT 0x0F EMIT 0x6E EMIT 0x5E EMIT 0x1C \
	799	EMIT 0x0F EMIT 0xF4 EMIT 0xD8 \
	800	EMIT 0x0F EMIT 0xD4 EMIT 0xCD \
	801	EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x10 \
	802	EMIT 0x0F EMIT 0xD4 EMIT 0xD5 \
	803	EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x08 \
	804	EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
	805	EMIT 0x0F EMIT 0xD4 EMIT 0xCF \
	806	EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x14 \
	807	EMIT 0x0F EMIT 0xD4 EMIT 0xE5 \
	808	EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x0C \
	809	EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
	810	EMIT 0x0F EMIT 0xD4 EMIT 0xCA \
	811	EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x18 \
	812	EMIT 0x0F EMIT 0xD4 EMIT 0xF5 \
	813	EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x10 \
	814	EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
	815	EMIT 0x0F EMIT 0xD4 EMIT 0xCC \
	816	EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x1C \
	817	EMIT 0x0F EMIT 0xD4 EMIT 0xDD \
	818	EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x14 \
	819	EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
	820	EMIT 0x0F EMIT 0xD4 EMIT 0xCE \
	821	EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x18 \
	822	EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
	823	EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
	824	EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x1C \
	825	EMIT 0x83 EMIT 0xC7 EMIT 0x20 \
	826	EMIT 0x83 EMIT 0xC6 EMIT 0x20 \
	827	EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
	828	EMIT 0x0F EMIT 0x7E EMIT 0xC9
	829
	830	#define MULADDC_STOP \
	831	EMIT 0x0F EMIT 0x77 \
	832	__asm mov c, ecx \
	833	__asm mov d, edi \
	834	__asm mov s, esi \
	835
	836	#else
	837
	838	#define MULADDC_STOP \
	839	__asm mov c, ecx \
	840	__asm mov d, edi \
	841	__asm mov s, esi \
	842
	843	#endif /* SSE2 */
	844	#endif /* MSVC */
	845
	846	#endif /* MBEDTLS_HAVE_ASM */
	847
	848	#if !defined(MULADDC_CORE)
	849	#if defined(MBEDTLS_HAVE_UDBL)
	850
	851	#define MULADDC_INIT \
	852	{ \
	853	mbedtls_t_udbl r; \
	854	mbedtls_mpi_uint r0, r1;
	855
	856	#define MULADDC_CORE \
	857	r = (s++) (mbedtls_t_udbl) b; \
	858	r0 = (mbedtls_mpi_uint) r; \
	859	r1 = (mbedtls_mpi_uint)( r >> biL ); \
	860	r0 += c; r1 += (r0 < c); \
	861	r0 += d; r1 += (r0 < d); \
	862	c = r1; *(d++) = r0;
	863
	864	#define MULADDC_STOP \
	865	}
	866
	867	#else
	868	#define MULADDC_INIT \
	869	{ \
	870	mbedtls_mpi_uint s0, s1, b0, b1; \
	871	mbedtls_mpi_uint r0, r1, rx, ry; \
	872	b0 = ( b << biH ) >> biH; \
	873	b1 = ( b >> biH );
	874
	875	#define MULADDC_CORE \
	876	s0 = ( *s << biH ) >> biH; \
	877	s1 = ( *s >> biH ); s++; \
	878	rx = s0 * b1; r0 = s0 * b0; \
	879	ry = s1 * b0; r1 = s1 * b1; \
	880	r1 += ( rx >> biH ); \
	881	r1 += ( ry >> biH ); \
	882	rx <<= biH; ry <<= biH; \
	883	r0 += rx; r1 += (r0 < rx); \
	884	r0 += ry; r1 += (r0 < ry); \
	885	r0 += c; r1 += (r0 < c); \
	886	r0 += d; r1 += (r0 < d); \
	887	c = r1; *(d++) = r0;
	888
	889	#define MULADDC_STOP \
	890	}
	891
	892	#endif /* C (generic) */
	893	#endif /* C (longlong) */
	894
	895	#endif /* bn_mul.h */