libxext libx11 libxcb libXinerama libXau pixman xserver git update 18/10/2010

11 files changed, 2286 insertions, 2077 deletions

diff --git a/pixman/configure.ac b/pixman/configure.ac
index ae7b4ee15..be1690b3b 100644
--- a/pixman/configure.ac
+++ b/pixman/configure.ac
@@ -600,13 +600,23 @@ AC_SUBST(DEP_CFLAGS)
 AC_SUBST(DEP_LIBS)

 

 dnl =====================================

-dnl posix_memalign 

+dnl posix_memalign, sigaction, alarm

 

 AC_CHECK_FUNC(posix_memalign, have_posix_memalign=yes, have_posix_memalign=no)

 if test x$have_posix_memalign = xyes; then

    AC_DEFINE(HAVE_POSIX_MEMALIGN, 1, [Whether we have posix_memalign()])

 fi

 

+AC_CHECK_FUNC(sigaction, have_sigaction=yes, have_sigaction=no)

+if test x$have_sigaction = xyes; then

+   AC_DEFINE(HAVE_SIGACTION, 1, [Whether we have sigaction()])

+fi

+

+AC_CHECK_FUNC(alarm, have_alarm=yes, have_alarm=no)

+if test x$have_alarm = xyes; then

+   AC_DEFINE(HAVE_ALARM, 1, [Whether we have alarm()])

+fi

+

 dnl =====================================

 dnl Thread local storage

 

diff --git a/pixman/pixman/pixman-arm-neon-asm.S b/pixman/pixman/pixman-arm-neon-asm.S
index 51bc347bc..18dca54ef 100644
--- a/pixman/pixman/pixman-arm-neon-asm.S
+++ b/pixman/pixman/pixman-arm-neon-asm.S
@@ -1,1713 +1,1806 @@
-/*
- * Copyright © 2009 Nokia Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Author:  Siarhei Siamashka (siarhei.siamashka@nokia.com)
- */
-
-/*
- * This file contains implementations of NEON optimized pixel processing
- * functions. There is no full and detailed tutorial, but some functions
- * (those which are exposing some new or interesting features) are
- * extensively commented and can be used as examples.
- *
- * You may want to have a look at the comments for following functions:
- *  - pixman_composite_over_8888_0565_asm_neon
- *  - pixman_composite_over_n_8_0565_asm_neon
- */
-
-/* Prevent the stack from becoming executable for no reason... */
-#if defined(__linux__) && defined(__ELF__)
-.section .note.GNU-stack,"",%progbits
-#endif
-
-    .text
-    .fpu neon
-    .arch armv7a
-    .object_arch armv4
-    .eabi_attribute 10, 0 /* suppress Tag_FP_arch */
-    .eabi_attribute 12, 0 /* suppress Tag_Advanced_SIMD_arch */
-    .arm
-    .altmacro
-
-#include "pixman-arm-neon-asm.h"
-
-/* Global configuration options and preferences */
-
-/*
- * The code can optionally make use of unaligned memory accesses to improve
- * performance of handling leading/trailing pixels for each scanline.
- * Configuration variable RESPECT_STRICT_ALIGNMENT can be set to 0 for
- * example in linux if unaligned memory accesses are not configured to
- * generate.exceptions.
- */
-.set RESPECT_STRICT_ALIGNMENT, 1
-
-/*
- * Set default prefetch type. There is a choice between the following options:
- *
- * PREFETCH_TYPE_NONE (may be useful for the ARM cores where PLD is set to work
- * as NOP to workaround some HW bugs or for whatever other reason)
- *
- * PREFETCH_TYPE_SIMPLE (may be useful for simple single-issue ARM cores where
- * advanced prefetch intruduces heavy overhead)
- *
- * PREFETCH_TYPE_ADVANCED (useful for superscalar cores such as ARM Cortex-A8
- * which can run ARM and NEON instructions simultaneously so that extra ARM
- * instructions do not add (many) extra cycles, but improve prefetch efficiency)
- *
- * Note: some types of function can't support advanced prefetch and fallback
- *       to simple one (those which handle 24bpp pixels)
- */
-.set PREFETCH_TYPE_DEFAULT, PREFETCH_TYPE_ADVANCED
-
-/* Prefetch distance in pixels for simple prefetch */
-.set PREFETCH_DISTANCE_SIMPLE, 64
-
-/*
- * Implementation of pixman_composite_over_8888_0565_asm_neon
- *
- * This function takes a8r8g8b8 source buffer, r5g6b5 destination buffer and
- * performs OVER compositing operation. Function fast_composite_over_8888_0565
- * from pixman-fast-path.c does the same in C and can be used as a reference.
- *
- * First we need to have some NEON assembly code which can do the actual
- * operation on the pixels and provide it to the template macro.
- *
- * Template macro quite conveniently takes care of emitting all the necessary
- * code for memory reading and writing (including quite tricky cases of
- * handling unaligned leading/trailing pixels), so we only need to deal with
- * the data in NEON registers.
- *
- * NEON registers allocation in general is recommented to be the following:
- * d0,  d1,  d2,  d3  - contain loaded source pixel data
- * d4,  d5,  d6,  d7  - contain loaded destination pixels (if they are needed)
- * d24, d25, d26, d27 - contain loading mask pixel data (if mask is used)
- * d28, d29, d30, d31 - place for storing the result (destination pixels)
- *
- * As can be seen above, four 64-bit NEON registers are used for keeping
- * intermediate pixel data and up to 8 pixels can be processed in one step
- * for 32bpp formats (16 pixels for 16bpp, 32 pixels for 8bpp).
- *
- * This particular function uses the following registers allocation:
- * d0,  d1,  d2,  d3  - contain loaded source pixel data
- * d4,  d5            - contain loaded destination pixels (they are needed)
- * d28, d29           - place for storing the result (destination pixels)
- */
-
-/*
- * Step one. We need to have some code to do some arithmetics on pixel data.
- * This is implemented as a pair of macros: '*_head' and '*_tail'. When used
- * back-to-back, they take pixel data from {d0, d1, d2, d3} and {d4, d5},
- * perform all the needed calculations and write the result to {d28, d29}.
- * The rationale for having two macros and not just one will be explained
- * later. In practice, any single monolitic function which does the work can
- * be split into two parts in any arbitrary way without affecting correctness.
- *
- * There is one special trick here too. Common template macro can optionally
- * make our life a bit easier by doing R, G, B, A color components
- * deinterleaving for 32bpp pixel formats (and this feature is used in
- * 'pixman_composite_over_8888_0565_asm_neon' function). So it means that
- * instead of having 8 packed pixels in {d0, d1, d2, d3} registers, we
- * actually use d0 register for blue channel (a vector of eight 8-bit
- * values), d1 register for green, d2 for red and d3 for alpha. This
- * simple conversion can be also done with a few NEON instructions:
- *
- * Packed to planar conversion:
- *  vuzp.8 d0, d1
- *  vuzp.8 d2, d3
- *  vuzp.8 d1, d3
- *  vuzp.8 d0, d2
- *
- * Planar to packed conversion:
- *  vzip.8 d0, d2
- *  vzip.8 d1, d3
- *  vzip.8 d2, d3
- *  vzip.8 d0, d1
- *
- * But pixel can be loaded directly in planar format using VLD4.8 NEON
- * instruction. It is 1 cycle slower than VLD1.32, so this is not always
- * desirable, that's why deinterleaving is optional.
- *
- * But anyway, here is the code:
- */
-.macro pixman_composite_over_8888_0565_process_pixblock_head
-    /* convert 8 r5g6b5 pixel data from {d4, d5} to planar 8-bit format
-       and put data into d6 - red, d7 - green, d30 - blue */
-    vshrn.u16   d6, q2, #8
-    vshrn.u16   d7, q2, #3
-    vsli.u16    q2, q2, #5
-    vsri.u8     d6, d6, #5
-    vmvn.8      d3, d3      /* invert source alpha */
-    vsri.u8     d7, d7, #6
-    vshrn.u16   d30, q2, #2
-    /* now do alpha blending, storing results in 8-bit planar format
-       into d16 - red, d19 - green, d18 - blue */
-    vmull.u8    q10, d3, d6
-    vmull.u8    q11, d3, d7
-    vmull.u8    q12, d3, d30
-    vrshr.u16   q13, q10, #8
-    vrshr.u16   q3, q11, #8
-    vrshr.u16   q15, q12, #8
-    vraddhn.u16 d20, q10, q13
-    vraddhn.u16 d23, q11, q3
-    vraddhn.u16 d22, q12, q15
-.endm
-
-.macro pixman_composite_over_8888_0565_process_pixblock_tail
-    /* ... continue alpha blending */
-    vqadd.u8    d16, d2, d20
-    vqadd.u8    q9, q0, q11
-    /* convert the result to r5g6b5 and store it into {d28, d29} */
-    vshll.u8    q14, d16, #8
-    vshll.u8    q8, d19, #8
-    vshll.u8    q9, d18, #8
-    vsri.u16    q14, q8, #5
-    vsri.u16    q14, q9, #11
-.endm
-
-/*
- * OK, now we got almost everything that we need. Using the above two
- * macros, the work can be done right. But now we want to optimize
- * it a bit. ARM Cortex-A8 is an in-order core, and benefits really
- * a lot from good code scheduling and software pipelining.
- *
- * Let's construct some code, which will run in the core main loop.
- * Some pseudo-code of the main loop will look like this:
- *   head
- *   while (...) {
- *     tail
- *     head
- *   }
- *   tail
- *
- * It may look a bit weird, but this setup allows to hide instruction
- * latencies better and also utilize dual-issue capability more
- * efficiently (make pairs of load-store and ALU instructions).
- *
- * So what we need now is a '*_tail_head' macro, which will be used
- * in the core main loop. A trivial straightforward implementation
- * of this macro would look like this:
- *
- *   pixman_composite_over_8888_0565_process_pixblock_tail
- *   vst1.16     {d28, d29}, [DST_W, :128]!
- *   vld1.16     {d4, d5}, [DST_R, :128]!
- *   vld4.32     {d0, d1, d2, d3}, [SRC]!
- *   pixman_composite_over_8888_0565_process_pixblock_head
- *   cache_preload 8, 8
- *
- * Now it also got some VLD/VST instructions. We simply can't move from
- * processing one block of pixels to the other one with just arithmetics.
- * The previously processed data needs to be written to memory and new
- * data needs to be fetched. Fortunately, this main loop does not deal
- * with partial leading/trailing pixels and can load/store a full block
- * of pixels in a bulk. Additionally, destination buffer is already
- * 16 bytes aligned here (which is good for performance).
- *
- * New things here are DST_R, DST_W, SRC and MASK identifiers. These
- * are the aliases for ARM registers which are used as pointers for
- * accessing data. We maintain separate pointers for reading and writing
- * destination buffer (DST_R and DST_W).
- *
- * Another new thing is 'cache_preload' macro. It is used for prefetching
- * data into CPU L2 cache and improve performance when dealing with large
- * images which are far larger than cache size. It uses one argument
- * (actually two, but they need to be the same here) - number of pixels
- * in a block. Looking into 'pixman-arm-neon-asm.h' can provide some
- * details about this macro. Moreover, if good performance is needed
- * the code from this macro needs to be copied into '*_tail_head' macro
- * and mixed with the rest of code for optimal instructions scheduling.
- * We are actually doing it below.
- *
- * Now after all the explanations, here is the optimized code.
- * Different instruction streams (originaling from '*_head', '*_tail'
- * and 'cache_preload' macro) use different indentation levels for
- * better readability. Actually taking the code from one of these
- * indentation levels and ignoring a few VLD/VST instructions would
- * result in exactly the code from '*_head', '*_tail' or 'cache_preload'
- * macro!
- */
-
-#if 1
-
-.macro pixman_composite_over_8888_0565_process_pixblock_tail_head
-        vqadd.u8    d16, d2, d20
-    vld1.16     {d4, d5}, [DST_R, :128]!
-        vqadd.u8    q9, q0, q11
-    vshrn.u16   d6, q2, #8
-    vld4.8      {d0, d1, d2, d3}, [SRC]!
-    vshrn.u16   d7, q2, #3
-    vsli.u16    q2, q2, #5
-        vshll.u8    q14, d16, #8
-                                    PF add PF_X, PF_X, #8
-        vshll.u8    q8, d19, #8
-                                    PF tst PF_CTL, #0xF
-    vsri.u8     d6, d6, #5
-                                    PF addne PF_X, PF_X, #8
-    vmvn.8      d3, d3
-                                    PF subne PF_CTL, PF_CTL, #1
-    vsri.u8     d7, d7, #6
-    vshrn.u16   d30, q2, #2
-    vmull.u8    q10, d3, d6
-                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
-    vmull.u8    q11, d3, d7
-    vmull.u8    q12, d3, d30
-                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
-        vsri.u16    q14, q8, #5
-                                    PF cmp PF_X, ORIG_W
-        vshll.u8    q9, d18, #8
-    vrshr.u16   q13, q10, #8
-                                    PF subge PF_X, PF_X, ORIG_W
-    vrshr.u16   q3, q11, #8
-    vrshr.u16   q15, q12, #8
-                                    PF subges PF_CTL, PF_CTL, #0x10
-        vsri.u16    q14, q9, #11
-                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
-    vraddhn.u16 d20, q10, q13
-    vraddhn.u16 d23, q11, q3
-                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
-    vraddhn.u16 d22, q12, q15
-        vst1.16     {d28, d29}, [DST_W, :128]!
-.endm
-
-#else
-
-/* If we did not care much about the performance, we would just use this... */
-.macro pixman_composite_over_8888_0565_process_pixblock_tail_head
-    pixman_composite_over_8888_0565_process_pixblock_tail
-    vst1.16     {d28, d29}, [DST_W, :128]!
-    vld1.16     {d4, d5}, [DST_R, :128]!
-    vld4.32     {d0, d1, d2, d3}, [SRC]!
-    pixman_composite_over_8888_0565_process_pixblock_head
-    cache_preload 8, 8
-.endm
-
-#endif
-
-/*
- * And now the final part. We are using 'generate_composite_function' macro
- * to put all the stuff together. We are specifying the name of the function
- * which we want to get, number of bits per pixel for the source, mask and
- * destination (0 if unused, like mask in this case). Next come some bit
- * flags:
- *   FLAG_DST_READWRITE      - tells that the destination buffer is both read
- *                             and written, for write-only buffer we would use
- *                             FLAG_DST_WRITEONLY flag instead
- *   FLAG_DEINTERLEAVE_32BPP - tells that we prefer to work with planar data
- *                             and separate color channels for 32bpp format.
- * The next things are:
- *  - the number of pixels processed per iteration (8 in this case, because
- *    that's the maximum what can fit into four 64-bit NEON registers).
- *  - prefetch distance, measured in pixel blocks. In this case it is 5 times
- *    by 8 pixels. That would be 40 pixels, or up to 160 bytes. Optimal
- *    prefetch distance can be selected by running some benchmarks.
- *
- * After that we specify some macros, these are 'default_init',
- * 'default_cleanup' here which are empty (but it is possible to have custom
- * init/cleanup macros to be able to save/restore some extra NEON registers
- * like d8-d15 or do anything else) followed by
- * 'pixman_composite_over_8888_0565_process_pixblock_head',
- * 'pixman_composite_over_8888_0565_process_pixblock_tail' and
- * 'pixman_composite_over_8888_0565_process_pixblock_tail_head'
- * which we got implemented above.
- *
- * The last part is the NEON registers allocation scheme.
- */
-generate_composite_function \
-    pixman_composite_over_8888_0565_asm_neon, 32, 0, 16, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_over_8888_0565_process_pixblock_head, \
-    pixman_composite_over_8888_0565_process_pixblock_tail, \
-    pixman_composite_over_8888_0565_process_pixblock_tail_head, \
-    28, /* dst_w_basereg */ \
-    4,  /* dst_r_basereg */ \
-    0,  /* src_basereg   */ \
-    24  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_over_n_0565_process_pixblock_head
-    /* convert 8 r5g6b5 pixel data from {d4, d5} to planar 8-bit format
-       and put data into d6 - red, d7 - green, d30 - blue */
-    vshrn.u16   d6, q2, #8
-    vshrn.u16   d7, q2, #3
-    vsli.u16    q2, q2, #5
-    vsri.u8     d6, d6, #5
-    vsri.u8     d7, d7, #6
-    vshrn.u16   d30, q2, #2
-    /* now do alpha blending, storing results in 8-bit planar format
-       into d16 - red, d19 - green, d18 - blue */
-    vmull.u8    q10, d3, d6
-    vmull.u8    q11, d3, d7
-    vmull.u8    q12, d3, d30
-    vrshr.u16   q13, q10, #8
-    vrshr.u16   q3, q11, #8
-    vrshr.u16   q15, q12, #8
-    vraddhn.u16 d20, q10, q13
-    vraddhn.u16 d23, q11, q3
-    vraddhn.u16 d22, q12, q15
-.endm
-
-.macro pixman_composite_over_n_0565_process_pixblock_tail
-    /* ... continue alpha blending */
-    vqadd.u8    d16, d2, d20
-    vqadd.u8    q9, q0, q11
-    /* convert the result to r5g6b5 and store it into {d28, d29} */
-    vshll.u8    q14, d16, #8
-    vshll.u8    q8, d19, #8
-    vshll.u8    q9, d18, #8
-    vsri.u16    q14, q8, #5
-    vsri.u16    q14, q9, #11
-.endm
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_over_n_0565_process_pixblock_tail_head
-    pixman_composite_over_n_0565_process_pixblock_tail
-    vld1.16     {d4, d5}, [DST_R, :128]!
-    vst1.16     {d28, d29}, [DST_W, :128]!
-    pixman_composite_over_n_0565_process_pixblock_head
-.endm
-
-.macro pixman_composite_over_n_0565_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vld1.32     {d3[0]}, [DUMMY]
-    vdup.8      d0, d3[0]
-    vdup.8      d1, d3[1]
-    vdup.8      d2, d3[2]
-    vdup.8      d3, d3[3]
-    vmvn.8      d3, d3      /* invert source alpha */
-.endm
-
-generate_composite_function \
-    pixman_composite_over_n_0565_asm_neon, 0, 0, 16, \
-    FLAG_DST_READWRITE, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_over_n_0565_init, \
-    default_cleanup, \
-    pixman_composite_over_n_0565_process_pixblock_head, \
-    pixman_composite_over_n_0565_process_pixblock_tail, \
-    pixman_composite_over_n_0565_process_pixblock_tail_head, \
-    28, /* dst_w_basereg */ \
-    4,  /* dst_r_basereg */ \
-    0,  /* src_basereg   */ \
-    24  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_8888_0565_process_pixblock_head
-    vshll.u8    q8, d1, #8
-    vshll.u8    q14, d2, #8
-    vshll.u8    q9, d0, #8
-.endm
-
-.macro pixman_composite_src_8888_0565_process_pixblock_tail
-    vsri.u16    q14, q8, #5
-    vsri.u16    q14, q9, #11
-.endm
-
-.macro pixman_composite_src_8888_0565_process_pixblock_tail_head
-        vsri.u16    q14, q8, #5
-                                    PF add PF_X, PF_X, #8
-                                    PF tst PF_CTL, #0xF
-    vld4.8      {d0, d1, d2, d3}, [SRC]!
-                                    PF addne PF_X, PF_X, #8
-                                    PF subne PF_CTL, PF_CTL, #1
-        vsri.u16    q14, q9, #11
-                                    PF cmp PF_X, ORIG_W
-                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
-    vshll.u8    q8, d1, #8
-        vst1.16     {d28, d29}, [DST_W, :128]!
-                                    PF subge PF_X, PF_X, ORIG_W
-                                    PF subges PF_CTL, PF_CTL, #0x10
-    vshll.u8    q14, d2, #8
-                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
-    vshll.u8    q9, d0, #8
-.endm
-
-generate_composite_function \
-    pixman_composite_src_8888_0565_asm_neon, 32, 0, 16, \
-    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_src_8888_0565_process_pixblock_head, \
-    pixman_composite_src_8888_0565_process_pixblock_tail, \
-    pixman_composite_src_8888_0565_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_src_0565_8888_process_pixblock_head
-    vshrn.u16   d30, q0, #8
-    vshrn.u16   d29, q0, #3
-    vsli.u16    q0, q0, #5
-    vmov.u8     d31, #255
-    vsri.u8     d30, d30, #5
-    vsri.u8     d29, d29, #6
-    vshrn.u16   d28, q0, #2
-.endm
-
-.macro pixman_composite_src_0565_8888_process_pixblock_tail
-.endm
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_src_0565_8888_process_pixblock_tail_head
-    pixman_composite_src_0565_8888_process_pixblock_tail
-    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
-    vld1.16    {d0, d1}, [SRC]!
-    pixman_composite_src_0565_8888_process_pixblock_head
-    cache_preload 8, 8
-.endm
-
-generate_composite_function \
-    pixman_composite_src_0565_8888_asm_neon, 16, 0, 32, \
-    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_src_0565_8888_process_pixblock_head, \
-    pixman_composite_src_0565_8888_process_pixblock_tail, \
-    pixman_composite_src_0565_8888_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_add_8000_8000_process_pixblock_head
-    vqadd.u8    q14, q0, q2
-    vqadd.u8    q15, q1, q3
-.endm
-
-.macro pixman_composite_add_8000_8000_process_pixblock_tail
-.endm
-
-.macro pixman_composite_add_8000_8000_process_pixblock_tail_head
-    vld1.8      {d0, d1, d2, d3}, [SRC]!
-                                    PF add PF_X, PF_X, #32
-                                    PF tst PF_CTL, #0xF
-    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
-                                    PF addne PF_X, PF_X, #32
-                                    PF subne PF_CTL, PF_CTL, #1
-        vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
-                                    PF cmp PF_X, ORIG_W
-                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
-                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
-                                    PF subge PF_X, PF_X, ORIG_W
-                                    PF subges PF_CTL, PF_CTL, #0x10
-    vqadd.u8    q14, q0, q2
-                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
-                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
-    vqadd.u8    q15, q1, q3
-.endm
-
-generate_composite_function \
-    pixman_composite_add_8000_8000_asm_neon, 8, 0, 8, \
-    FLAG_DST_READWRITE, \
-    32, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_add_8000_8000_process_pixblock_head, \
-    pixman_composite_add_8000_8000_process_pixblock_tail, \
-    pixman_composite_add_8000_8000_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_add_8888_8888_process_pixblock_tail_head
-    vld1.8      {d0, d1, d2, d3}, [SRC]!
-                                    PF add PF_X, PF_X, #8
-                                    PF tst PF_CTL, #0xF
-    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
-                                    PF addne PF_X, PF_X, #8
-                                    PF subne PF_CTL, PF_CTL, #1
-        vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
-                                    PF cmp PF_X, ORIG_W
-                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
-                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
-                                    PF subge PF_X, PF_X, ORIG_W
-                                    PF subges PF_CTL, PF_CTL, #0x10
-    vqadd.u8    q14, q0, q2
-                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
-                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
-    vqadd.u8    q15, q1, q3
-.endm
-
-generate_composite_function \
-    pixman_composite_add_8888_8888_asm_neon, 32, 0, 32, \
-    FLAG_DST_READWRITE, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_add_8000_8000_process_pixblock_head, \
-    pixman_composite_add_8000_8000_process_pixblock_tail, \
-    pixman_composite_add_8888_8888_process_pixblock_tail_head
-
-generate_composite_function_single_scanline \
-    pixman_composite_scanline_add_asm_neon, 32, 0, 32, \
-    FLAG_DST_READWRITE, \
-    8, /* number of pixels, processed in a single block */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_add_8000_8000_process_pixblock_head, \
-    pixman_composite_add_8000_8000_process_pixblock_tail, \
-    pixman_composite_add_8888_8888_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_over_8888_8888_process_pixblock_head
-    vmvn.8      d24, d3  /* get inverted alpha */
-    /* do alpha blending */
-    vmull.u8    q8, d24, d4
-    vmull.u8    q9, d24, d5
-    vmull.u8    q10, d24, d6
-    vmull.u8    q11, d24, d7
-.endm
-
-.macro pixman_composite_over_8888_8888_process_pixblock_tail
-    vrshr.u16   q14, q8, #8
-    vrshr.u16   q15, q9, #8
-    vrshr.u16   q12, q10, #8
-    vrshr.u16   q13, q11, #8
-    vraddhn.u16 d28, q14, q8
-    vraddhn.u16 d29, q15, q9
-    vraddhn.u16 d30, q12, q10
-    vraddhn.u16 d31, q13, q11
-    vqadd.u8    q14, q0, q14
-    vqadd.u8    q15, q1, q15
-.endm
-
-.macro pixman_composite_over_8888_8888_process_pixblock_tail_head
-    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
-        vrshr.u16   q14, q8, #8
-                                    PF add PF_X, PF_X, #8
-                                    PF tst PF_CTL, #0xF
-        vrshr.u16   q15, q9, #8
-        vrshr.u16   q12, q10, #8
-        vrshr.u16   q13, q11, #8
-                                    PF addne PF_X, PF_X, #8
-                                    PF subne PF_CTL, PF_CTL, #1
-        vraddhn.u16 d28, q14, q8
-        vraddhn.u16 d29, q15, q9
-                                    PF cmp PF_X, ORIG_W
-        vraddhn.u16 d30, q12, q10
-        vraddhn.u16 d31, q13, q11
-        vqadd.u8    q14, q0, q14
-        vqadd.u8    q15, q1, q15
-    vld4.8      {d0, d1, d2, d3}, [SRC]!
-                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
-    vmvn.8      d22, d3
-                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
-        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
-                                    PF subge PF_X, PF_X, ORIG_W
-    vmull.u8    q8, d22, d4
-                                    PF subges PF_CTL, PF_CTL, #0x10
-    vmull.u8    q9, d22, d5
-                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
-    vmull.u8    q10, d22, d6
-                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
-    vmull.u8    q11, d22, d7
-.endm
-
-generate_composite_function \
-    pixman_composite_over_8888_8888_asm_neon, 32, 0, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_over_8888_8888_process_pixblock_head, \
-    pixman_composite_over_8888_8888_process_pixblock_tail, \
-    pixman_composite_over_8888_8888_process_pixblock_tail_head
-
-generate_composite_function_single_scanline \
-    pixman_composite_scanline_over_asm_neon, 32, 0, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_over_8888_8888_process_pixblock_head, \
-    pixman_composite_over_8888_8888_process_pixblock_tail, \
-    pixman_composite_over_8888_8888_process_pixblock_tail_head
-
-/******************************************************************************/
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_over_n_8888_process_pixblock_tail_head
-    pixman_composite_over_8888_8888_process_pixblock_tail
-    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
-    pixman_composite_over_8888_8888_process_pixblock_head
-.endm
-
-.macro pixman_composite_over_n_8888_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vld1.32     {d3[0]}, [DUMMY]
-    vdup.8      d0, d3[0]
-    vdup.8      d1, d3[1]
-    vdup.8      d2, d3[2]
-    vdup.8      d3, d3[3]
-.endm
-
-generate_composite_function \
-    pixman_composite_over_n_8888_asm_neon, 0, 0, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_over_n_8888_init, \
-    default_cleanup, \
-    pixman_composite_over_8888_8888_process_pixblock_head, \
-    pixman_composite_over_8888_8888_process_pixblock_tail, \
-    pixman_composite_over_n_8888_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_over_reverse_n_8888_process_pixblock_tail_head
-        vrshr.u16   q14, q8, #8
-                                    PF add PF_X, PF_X, #8
-                                    PF tst PF_CTL, #0xF
-        vrshr.u16   q15, q9, #8
-        vrshr.u16   q12, q10, #8
-        vrshr.u16   q13, q11, #8
-                                    PF addne PF_X, PF_X, #8
-                                    PF subne PF_CTL, PF_CTL, #1
-        vraddhn.u16 d28, q14, q8
-        vraddhn.u16 d29, q15, q9
-                                    PF cmp PF_X, ORIG_W
-        vraddhn.u16 d30, q12, q10
-        vraddhn.u16 d31, q13, q11
-        vqadd.u8    q14, q0, q14
-        vqadd.u8    q15, q1, q15
-    vld4.8      {d0, d1, d2, d3}, [DST_R, :128]!
-    vmvn.8      d22, d3
-                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
-        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
-                                    PF subge PF_X, PF_X, ORIG_W
-    vmull.u8    q8, d22, d4
-                                    PF subges PF_CTL, PF_CTL, #0x10
-    vmull.u8    q9, d22, d5
-    vmull.u8    q10, d22, d6
-                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
-    vmull.u8    q11, d22, d7
-.endm
-
-.macro pixman_composite_over_reverse_n_8888_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vld1.32     {d7[0]}, [DUMMY]
-    vdup.8      d4, d7[0]
-    vdup.8      d5, d7[1]
-    vdup.8      d6, d7[2]
-    vdup.8      d7, d7[3]
-.endm
-
-generate_composite_function \
-    pixman_composite_over_reverse_n_8888_asm_neon, 0, 0, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_over_reverse_n_8888_init, \
-    default_cleanup, \
-    pixman_composite_over_8888_8888_process_pixblock_head, \
-    pixman_composite_over_8888_8888_process_pixblock_tail, \
-    pixman_composite_over_reverse_n_8888_process_pixblock_tail_head, \
-    28, /* dst_w_basereg */ \
-    0,  /* dst_r_basereg */ \
-    4,  /* src_basereg   */ \
-    24  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_over_n_8_0565_process_pixblock_head
-    /* in */
-    vmull.u8    q0, d24, d8
-    vmull.u8    q1, d24, d9
-    vmull.u8    q6, d24, d10
-    vmull.u8    q7, d24, d11
-    vrshr.u16   q10, q0, #8
-    vrshr.u16   q11, q1, #8
-    vrshr.u16   q12, q6, #8
-    vrshr.u16   q13, q7, #8
-    vraddhn.u16 d0, q0, q10
-    vraddhn.u16 d1, q1, q11
-    vraddhn.u16 d2, q6, q12
-    vraddhn.u16 d3, q7, q13
-
-    vshrn.u16   d6, q2, #8
-    vshrn.u16   d7, q2, #3
-    vsli.u16    q2, q2, #5
-    vsri.u8     d6, d6, #5
-    vmvn.8      d3, d3
-    vsri.u8     d7, d7, #6
-    vshrn.u16   d30, q2, #2
-    /* now do alpha blending */
-    vmull.u8    q10, d3, d6
-    vmull.u8    q11, d3, d7
-    vmull.u8    q12, d3, d30
-    vrshr.u16   q13, q10, #8
-    vrshr.u16   q3, q11, #8
-    vrshr.u16   q15, q12, #8
-    vraddhn.u16 d20, q10, q13
-    vraddhn.u16 d23, q11, q3
-    vraddhn.u16 d22, q12, q15
-.endm
-
-.macro pixman_composite_over_n_8_0565_process_pixblock_tail
-    vqadd.u8    d16, d2, d20
-    vqadd.u8    q9, q0, q11
-    /* convert to r5g6b5 */
-    vshll.u8    q14, d16, #8
-    vshll.u8    q8, d19, #8
-    vshll.u8    q9, d18, #8
-    vsri.u16    q14, q8, #5
-    vsri.u16    q14, q9, #11
-.endm
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_over_n_8_0565_process_pixblock_tail_head
-    pixman_composite_over_n_8_0565_process_pixblock_tail
-    vst1.16     {d28, d29}, [DST_W, :128]!
-    vld1.16     {d4, d5}, [DST_R, :128]!
-    vld1.8      {d24}, [MASK]!
-    cache_preload 8, 8
-    pixman_composite_over_n_8_0565_process_pixblock_head
-.endm
-
-/*
- * This function needs a special initialization of solid mask.
- * Solid source pixel data is fetched from stack at ARGS_STACK_OFFSET
- * offset, split into color components and replicated in d8-d11
- * registers. Additionally, this function needs all the NEON registers,
- * so it has to save d8-d15 registers which are callee saved according
- * to ABI. These registers are restored from 'cleanup' macro. All the
- * other NEON registers are caller saved, so can be clobbered freely
- * without introducing any problems.
- */
-.macro pixman_composite_over_n_8_0565_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vpush       {d8-d15}
-    vld1.32     {d11[0]}, [DUMMY]
-    vdup.8      d8, d11[0]
-    vdup.8      d9, d11[1]
-    vdup.8      d10, d11[2]
-    vdup.8      d11, d11[3]
-.endm
-
-.macro pixman_composite_over_n_8_0565_cleanup
-    vpop        {d8-d15}
-.endm
-
-generate_composite_function \
-    pixman_composite_over_n_8_0565_asm_neon, 0, 8, 16, \
-    FLAG_DST_READWRITE, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_over_n_8_0565_init, \
-    pixman_composite_over_n_8_0565_cleanup, \
-    pixman_composite_over_n_8_0565_process_pixblock_head, \
-    pixman_composite_over_n_8_0565_process_pixblock_tail, \
-    pixman_composite_over_n_8_0565_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_src_0565_0565_process_pixblock_head
-.endm
-
-.macro pixman_composite_src_0565_0565_process_pixblock_tail
-.endm
-
-.macro pixman_composite_src_0565_0565_process_pixblock_tail_head
-    vst1.16 {d0, d1, d2, d3}, [DST_W, :128]!
-    vld1.16 {d0, d1, d2, d3}, [SRC]!
-    cache_preload 16, 16
-.endm
-
-generate_composite_function \
-    pixman_composite_src_0565_0565_asm_neon, 16, 0, 16, \
-    FLAG_DST_WRITEONLY, \
-    16, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_src_0565_0565_process_pixblock_head, \
-    pixman_composite_src_0565_0565_process_pixblock_tail, \
-    pixman_composite_src_0565_0565_process_pixblock_tail_head, \
-    0, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_n_8_process_pixblock_head
-.endm
-
-.macro pixman_composite_src_n_8_process_pixblock_tail
-.endm
-
-.macro pixman_composite_src_n_8_process_pixblock_tail_head
-    vst1.8  {d0, d1, d2, d3}, [DST_W, :128]!
-.endm
-
-.macro pixman_composite_src_n_8_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vld1.32     {d0[0]}, [DUMMY]
-    vsli.u64    d0, d0, #8
-    vsli.u64    d0, d0, #16
-    vsli.u64    d0, d0, #32
-    vorr        d1, d0, d0
-    vorr        q1, q0, q0
-.endm
-
-.macro pixman_composite_src_n_8_cleanup
-.endm
-
-generate_composite_function \
-    pixman_composite_src_n_8_asm_neon, 0, 0, 8, \
-    FLAG_DST_WRITEONLY, \
-    32, /* number of pixels, processed in a single block */ \
-    0,  /* prefetch distance */ \
-    pixman_composite_src_n_8_init, \
-    pixman_composite_src_n_8_cleanup, \
-    pixman_composite_src_n_8_process_pixblock_head, \
-    pixman_composite_src_n_8_process_pixblock_tail, \
-    pixman_composite_src_n_8_process_pixblock_tail_head, \
-    0, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_n_0565_process_pixblock_head
-.endm
-
-.macro pixman_composite_src_n_0565_process_pixblock_tail
-.endm
-
-.macro pixman_composite_src_n_0565_process_pixblock_tail_head
-    vst1.16 {d0, d1, d2, d3}, [DST_W, :128]!
-.endm
-
-.macro pixman_composite_src_n_0565_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vld1.32     {d0[0]}, [DUMMY]
-    vsli.u64    d0, d0, #16
-    vsli.u64    d0, d0, #32
-    vorr        d1, d0, d0
-    vorr        q1, q0, q0
-.endm
-
-.macro pixman_composite_src_n_0565_cleanup
-.endm
-
-generate_composite_function \
-    pixman_composite_src_n_0565_asm_neon, 0, 0, 16, \
-    FLAG_DST_WRITEONLY, \
-    16, /* number of pixels, processed in a single block */ \
-    0,  /* prefetch distance */ \
-    pixman_composite_src_n_0565_init, \
-    pixman_composite_src_n_0565_cleanup, \
-    pixman_composite_src_n_0565_process_pixblock_head, \
-    pixman_composite_src_n_0565_process_pixblock_tail, \
-    pixman_composite_src_n_0565_process_pixblock_tail_head, \
-    0, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_n_8888_process_pixblock_head
-.endm
-
-.macro pixman_composite_src_n_8888_process_pixblock_tail
-.endm
-
-.macro pixman_composite_src_n_8888_process_pixblock_tail_head
-    vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!
-.endm
-
-.macro pixman_composite_src_n_8888_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vld1.32     {d0[0]}, [DUMMY]
-    vsli.u64    d0, d0, #32
-    vorr        d1, d0, d0
-    vorr        q1, q0, q0
-.endm
-
-.macro pixman_composite_src_n_8888_cleanup
-.endm
-
-generate_composite_function \
-    pixman_composite_src_n_8888_asm_neon, 0, 0, 32, \
-    FLAG_DST_WRITEONLY, \
-    8, /* number of pixels, processed in a single block */ \
-    0, /* prefetch distance */ \
-    pixman_composite_src_n_8888_init, \
-    pixman_composite_src_n_8888_cleanup, \
-    pixman_composite_src_n_8888_process_pixblock_head, \
-    pixman_composite_src_n_8888_process_pixblock_tail, \
-    pixman_composite_src_n_8888_process_pixblock_tail_head, \
-    0, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_8888_8888_process_pixblock_head
-.endm
-
-.macro pixman_composite_src_8888_8888_process_pixblock_tail
-.endm
-
-.macro pixman_composite_src_8888_8888_process_pixblock_tail_head
-    vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!
-    vld1.32 {d0, d1, d2, d3}, [SRC]!
-    cache_preload 8, 8
-.endm
-
-generate_composite_function \
-    pixman_composite_src_8888_8888_asm_neon, 32, 0, 32, \
-    FLAG_DST_WRITEONLY, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_src_8888_8888_process_pixblock_head, \
-    pixman_composite_src_8888_8888_process_pixblock_tail, \
-    pixman_composite_src_8888_8888_process_pixblock_tail_head, \
-    0, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_x888_8888_process_pixblock_head
-    vorr     q0, q0, q2
-    vorr     q1, q1, q2
-.endm
-
-.macro pixman_composite_src_x888_8888_process_pixblock_tail
-.endm
-
-.macro pixman_composite_src_x888_8888_process_pixblock_tail_head
-    vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!
-    vld1.32 {d0, d1, d2, d3}, [SRC]!
-    vorr     q0, q0, q2
-    vorr     q1, q1, q2
-    cache_preload 8, 8
-.endm
-
-.macro pixman_composite_src_x888_8888_init
-    vmov.u8  q2, #0xFF
-    vshl.u32 q2, q2, #24
-.endm
-
-generate_composite_function \
-    pixman_composite_src_x888_8888_asm_neon, 32, 0, 32, \
-    FLAG_DST_WRITEONLY, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    pixman_composite_src_x888_8888_init, \
-    default_cleanup, \
-    pixman_composite_src_x888_8888_process_pixblock_head, \
-    pixman_composite_src_x888_8888_process_pixblock_tail, \
-    pixman_composite_src_x888_8888_process_pixblock_tail_head, \
-    0, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_over_n_8_8888_process_pixblock_head
-    /* expecting deinterleaved source data in {d8, d9, d10, d11} */
-    /* d8 - blue, d9 - green, d10 - red, d11 - alpha */
-    /* and destination data in {d4, d5, d6, d7} */
-    /* mask is in d24 (d25, d26, d27 are unused) */
-
-    /* in */
-    vmull.u8    q0, d24, d8
-    vmull.u8    q1, d24, d9
-    vmull.u8    q6, d24, d10
-    vmull.u8    q7, d24, d11
-    vrshr.u16   q10, q0, #8
-    vrshr.u16   q11, q1, #8
-    vrshr.u16   q12, q6, #8
-    vrshr.u16   q13, q7, #8
-    vraddhn.u16 d0, q0, q10
-    vraddhn.u16 d1, q1, q11
-    vraddhn.u16 d2, q6, q12
-    vraddhn.u16 d3, q7, q13
-    vmvn.8      d24, d3  /* get inverted alpha */
-    /* source:      d0 - blue, d1 - green, d2 - red, d3 - alpha */
-    /* destination: d4 - blue, d5 - green, d6 - red, d7 - alpha */
-    /* now do alpha blending */
-    vmull.u8    q8, d24, d4
-    vmull.u8    q9, d24, d5
-    vmull.u8    q10, d24, d6
-    vmull.u8    q11, d24, d7
-.endm
-
-.macro pixman_composite_over_n_8_8888_process_pixblock_tail
-    vrshr.u16   q14, q8, #8
-    vrshr.u16   q15, q9, #8
-    vrshr.u16   q12, q10, #8
-    vrshr.u16   q13, q11, #8
-    vraddhn.u16 d28, q14, q8
-    vraddhn.u16 d29, q15, q9
-    vraddhn.u16 d30, q12, q10
-    vraddhn.u16 d31, q13, q11
-    vqadd.u8    q14, q0, q14
-    vqadd.u8    q15, q1, q15
-.endm
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_over_n_8_8888_process_pixblock_tail_head
-    pixman_composite_over_n_8_8888_process_pixblock_tail
-    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
-    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vld1.8      {d24}, [MASK]!
-    cache_preload 8, 8
-    pixman_composite_over_n_8_8888_process_pixblock_head
-.endm
-
-.macro pixman_composite_over_n_8_8888_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vpush       {d8-d15}
-    vld1.32     {d11[0]}, [DUMMY]
-    vdup.8      d8, d11[0]
-    vdup.8      d9, d11[1]
-    vdup.8      d10, d11[2]
-    vdup.8      d11, d11[3]
-.endm
-
-.macro pixman_composite_over_n_8_8888_cleanup
-    vpop        {d8-d15}
-.endm
-
-generate_composite_function \
-    pixman_composite_over_n_8_8888_asm_neon, 0, 8, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_over_n_8_8888_init, \
-    pixman_composite_over_n_8_8888_cleanup, \
-    pixman_composite_over_n_8_8888_process_pixblock_head, \
-    pixman_composite_over_n_8_8888_process_pixblock_tail, \
-    pixman_composite_over_n_8_8888_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_head
-    /*
-     * 'combine_mask_ca' replacement
-     *
-     * input:  solid src (n) in {d8,  d9,  d10, d11}
-     *         dest in          {d4,  d5,  d6,  d7 }
-     *         mask in          {d24, d25, d26, d27}
-     * output: updated src in   {d0,  d1,  d2,  d3 }
-     *         updated mask in  {d24, d25, d26, d3 }
-     */
-    vmull.u8    q0,  d24, d8
-    vmull.u8    q1,  d25, d9
-    vmull.u8    q6,  d26, d10
-    vmull.u8    q7,  d27, d11
-    vmull.u8    q9,  d11, d25
-    vmull.u8    q12, d11, d24
-    vmull.u8    q13, d11, d26
-    vrshr.u16   q8,  q0,  #8
-    vrshr.u16   q10, q1,  #8
-    vrshr.u16   q11, q6,  #8
-    vraddhn.u16 d0,  q0,  q8
-    vraddhn.u16 d1,  q1,  q10
-    vraddhn.u16 d2,  q6,  q11
-    vrshr.u16   q11, q12, #8
-    vrshr.u16   q8,  q9,  #8
-    vrshr.u16   q6,  q13, #8
-    vrshr.u16   q10, q7,  #8
-    vraddhn.u16 d24, q12, q11
-    vraddhn.u16 d25, q9,  q8
-    vraddhn.u16 d26, q13, q6
-    vraddhn.u16 d3,  q7,  q10
-    /*
-     * 'combine_over_ca' replacement
-     *
-     * output: updated dest in {d28, d29, d30, d31}
-     */
-    vmvn.8      d24, d24
-    vmvn.8      d25, d25
-    vmull.u8    q8,  d24, d4
-    vmull.u8    q9,  d25, d5
-    vmvn.8      d26, d26
-    vmvn.8      d27, d3
-    vmull.u8    q10, d26, d6
-    vmull.u8    q11, d27, d7
-.endm
-
-.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail
-    /* ... continue 'combine_over_ca' replacement */
-    vrshr.u16   q14, q8,  #8
-    vrshr.u16   q15, q9,  #8
-    vrshr.u16   q6,  q10, #8
-    vrshr.u16   q7,  q11, #8
-    vraddhn.u16 d28, q14, q8
-    vraddhn.u16 d29, q15, q9
-    vraddhn.u16 d30, q6,  q10
-    vraddhn.u16 d31, q7,  q11
-    vqadd.u8    q14, q0,  q14
-    vqadd.u8    q15, q1,  q15
-.endm
-
-.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head
-        vrshr.u16   q14, q8, #8
-        vrshr.u16   q15, q9, #8
-    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
-        vrshr.u16   q6, q10, #8
-        vrshr.u16   q7, q11, #8
-        vraddhn.u16 d28, q14, q8
-        vraddhn.u16 d29, q15, q9
-        vraddhn.u16 d30, q6, q10
-        vraddhn.u16 d31, q7, q11
-    vld4.8      {d24, d25, d26, d27}, [MASK]!
-        vqadd.u8    q14, q0, q14
-        vqadd.u8    q15, q1, q15
-    cache_preload 8, 8
-    pixman_composite_over_n_8888_8888_ca_process_pixblock_head
-    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
-.endm
-
-.macro pixman_composite_over_n_8888_8888_ca_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vpush       {d8-d15}
-    vld1.32     {d11[0]}, [DUMMY]
-    vdup.8      d8, d11[0]
-    vdup.8      d9, d11[1]
-    vdup.8      d10, d11[2]
-    vdup.8      d11, d11[3]
-.endm
-
-.macro pixman_composite_over_n_8888_8888_ca_cleanup
-    vpop        {d8-d15}
-.endm
-
-generate_composite_function \
-    pixman_composite_over_n_8888_8888_ca_asm_neon, 0, 32, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_over_n_8888_8888_ca_init, \
-    pixman_composite_over_n_8888_8888_ca_cleanup, \
-    pixman_composite_over_n_8888_8888_ca_process_pixblock_head, \
-    pixman_composite_over_n_8888_8888_ca_process_pixblock_tail, \
-    pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_add_n_8_8_process_pixblock_head
-    /* expecting source data in {d8, d9, d10, d11} */
-    /* d8 - blue, d9 - green, d10 - red, d11 - alpha */
-    /* and destination data in {d4, d5, d6, d7} */
-    /* mask is in d24, d25, d26, d27 */
-    vmull.u8    q0, d24, d11
-    vmull.u8    q1, d25, d11
-    vmull.u8    q6, d26, d11
-    vmull.u8    q7, d27, d11
-    vrshr.u16   q10, q0, #8
-    vrshr.u16   q11, q1, #8
-    vrshr.u16   q12, q6, #8
-    vrshr.u16   q13, q7, #8
-    vraddhn.u16 d0, q0, q10
-    vraddhn.u16 d1, q1, q11
-    vraddhn.u16 d2, q6, q12
-    vraddhn.u16 d3, q7, q13
-    vqadd.u8    q14, q0, q2
-    vqadd.u8    q15, q1, q3
-.endm
-
-.macro pixman_composite_add_n_8_8_process_pixblock_tail
-.endm
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_add_n_8_8_process_pixblock_tail_head
-    pixman_composite_add_n_8_8_process_pixblock_tail
-    vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
-    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vld1.8      {d24, d25, d26, d27}, [MASK]!
-    cache_preload 32, 32
-    pixman_composite_add_n_8_8_process_pixblock_head
-.endm
-
-.macro pixman_composite_add_n_8_8_init
-    add         DUMMY, sp, #ARGS_STACK_OFFSET
-    vpush       {d8-d15}
-    vld1.32     {d11[0]}, [DUMMY]
-    vdup.8      d11, d11[3]
-.endm
-
-.macro pixman_composite_add_n_8_8_cleanup
-    vpop        {d8-d15}
-.endm
-
-generate_composite_function \
-    pixman_composite_add_n_8_8_asm_neon, 0, 8, 8, \
-    FLAG_DST_READWRITE, \
-    32, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_add_n_8_8_init, \
-    pixman_composite_add_n_8_8_cleanup, \
-    pixman_composite_add_n_8_8_process_pixblock_head, \
-    pixman_composite_add_n_8_8_process_pixblock_tail, \
-    pixman_composite_add_n_8_8_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_add_8_8_8_process_pixblock_head
-    /* expecting source data in {d0, d1, d2, d3} */
-    /* destination data in {d4, d5, d6, d7} */
-    /* mask in {d24, d25, d26, d27} */
-    vmull.u8    q8, d24, d0
-    vmull.u8    q9, d25, d1
-    vmull.u8    q10, d26, d2
-    vmull.u8    q11, d27, d3
-    vrshr.u16   q0, q8, #8
-    vrshr.u16   q1, q9, #8
-    vrshr.u16   q12, q10, #8
-    vrshr.u16   q13, q11, #8
-    vraddhn.u16 d0, q0, q8
-    vraddhn.u16 d1, q1, q9
-    vraddhn.u16 d2, q12, q10
-    vraddhn.u16 d3, q13, q11
-    vqadd.u8    q14, q0, q2
-    vqadd.u8    q15, q1, q3
-.endm
-
-.macro pixman_composite_add_8_8_8_process_pixblock_tail
-.endm
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_add_8_8_8_process_pixblock_tail_head
-    pixman_composite_add_8_8_8_process_pixblock_tail
-    vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
-    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vld1.8      {d24, d25, d26, d27}, [MASK]!
-    vld1.8      {d0, d1, d2, d3}, [SRC]!
-    cache_preload 32, 32
-    pixman_composite_add_8_8_8_process_pixblock_head
-.endm
-
-.macro pixman_composite_add_8_8_8_init
-.endm
-
-.macro pixman_composite_add_8_8_8_cleanup
-.endm
-
-generate_composite_function \
-    pixman_composite_add_8_8_8_asm_neon, 8, 8, 8, \
-    FLAG_DST_READWRITE, \
-    32, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_add_8_8_8_init, \
-    pixman_composite_add_8_8_8_cleanup, \
-    pixman_composite_add_8_8_8_process_pixblock_head, \
-    pixman_composite_add_8_8_8_process_pixblock_tail, \
-    pixman_composite_add_8_8_8_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_add_8888_8888_8888_process_pixblock_head
-    /* expecting source data in {d0, d1, d2, d3} */
-    /* destination data in {d4, d5, d6, d7} */
-    /* mask in {d24, d25, d26, d27} */
-    vmull.u8    q8, d27, d0
-    vmull.u8    q9, d27, d1
-    vmull.u8    q10, d27, d2
-    vmull.u8    q11, d27, d3
-    vrshr.u16   q0, q8, #8
-    vrshr.u16   q1, q9, #8
-    vrshr.u16   q12, q10, #8
-    vrshr.u16   q13, q11, #8
-    vraddhn.u16 d0, q0, q8
-    vraddhn.u16 d1, q1, q9
-    vraddhn.u16 d2, q12, q10
-    vraddhn.u16 d3, q13, q11
-    vqadd.u8    q14, q0, q2
-    vqadd.u8    q15, q1, q3
-.endm
-
-.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail
-.endm
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail_head
-    pixman_composite_add_8888_8888_8888_process_pixblock_tail
-    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
-    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
-    vld4.8      {d24, d25, d26, d27}, [MASK]!
-    vld4.8      {d0, d1, d2, d3}, [SRC]!
-    cache_preload 8, 8
-    pixman_composite_add_8888_8888_8888_process_pixblock_head
-.endm
-
-generate_composite_function \
-    pixman_composite_add_8888_8888_8888_asm_neon, 32, 32, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
-    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
-    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head
-
-generate_composite_function_single_scanline \
-    pixman_composite_scanline_add_mask_asm_neon, 32, 32, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
-    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
-    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head
-
-/******************************************************************************/
-
-.macro pixman_composite_over_8888_n_8888_process_pixblock_head
-    /* expecting source data in {d0, d1, d2, d3} */
-    /* destination data in {d4, d5, d6, d7} */
-    /* solid mask is in d15 */
-
-    /* 'in' */
-    vmull.u8    q8, d15, d3
-    vmull.u8    q6, d15, d2
-    vmull.u8    q5, d15, d1
-    vmull.u8    q4, d15, d0
-    vrshr.u16   q13, q8, #8
-    vrshr.u16   q12, q6, #8
-    vrshr.u16   q11, q5, #8
-    vrshr.u16   q10, q4, #8
-    vraddhn.u16 d3, q8, q13
-    vraddhn.u16 d2, q6, q12
-    vraddhn.u16 d1, q5, q11
-    vraddhn.u16 d0, q4, q10
-    vmvn.8      d24, d3  /* get inverted alpha */
-    /* now do alpha blending */
-    vmull.u8    q8, d24, d4
-    vmull.u8    q9, d24, d5
-    vmull.u8    q10, d24, d6
-    vmull.u8    q11, d24, d7
-.endm
-
-.macro pixman_composite_over_8888_n_8888_process_pixblock_tail
-    vrshr.u16   q14, q8, #8
-    vrshr.u16   q15, q9, #8
-    vrshr.u16   q12, q10, #8
-    vrshr.u16   q13, q11, #8
-    vraddhn.u16 d28, q14, q8
-    vraddhn.u16 d29, q15, q9
-    vraddhn.u16 d30, q12, q10
-    vraddhn.u16 d31, q13, q11
-    vqadd.u8    q14, q0, q14
-    vqadd.u8    q15, q1, q15
-.endm
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_over_8888_n_8888_process_pixblock_tail_head
-    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
-    pixman_composite_over_8888_n_8888_process_pixblock_tail
-    vld4.8     {d0, d1, d2, d3}, [SRC]!
-    cache_preload 8, 8
-    pixman_composite_over_8888_n_8888_process_pixblock_head
-    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
-.endm
-
-.macro pixman_composite_over_8888_n_8888_init
-    add         DUMMY, sp, #48
-    vpush       {d8-d15}
-    vld1.32     {d15[0]}, [DUMMY]
-    vdup.8      d15, d15[3]
-.endm
-
-.macro pixman_composite_over_8888_n_8888_cleanup
-    vpop        {d8-d15}
-.endm
-
-generate_composite_function \
-    pixman_composite_over_8888_n_8888_asm_neon, 32, 0, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_over_8888_n_8888_init, \
-    pixman_composite_over_8888_n_8888_cleanup, \
-    pixman_composite_over_8888_n_8888_process_pixblock_head, \
-    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
-    pixman_composite_over_8888_n_8888_process_pixblock_tail_head
-
-/******************************************************************************/
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_over_8888_8888_8888_process_pixblock_tail_head
-    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
-    pixman_composite_over_8888_n_8888_process_pixblock_tail
-    vld4.8     {d0, d1, d2, d3}, [SRC]!
-    cache_preload 8, 8
-    vld4.8     {d12, d13, d14, d15}, [MASK]!
-    pixman_composite_over_8888_n_8888_process_pixblock_head
-    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
-.endm
-
-.macro pixman_composite_over_8888_8888_8888_init
-    vpush       {d8-d15}
-.endm
-
-.macro pixman_composite_over_8888_8888_8888_cleanup
-    vpop        {d8-d15}
-.endm
-
-generate_composite_function \
-    pixman_composite_over_8888_8888_8888_asm_neon, 32, 32, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_over_8888_8888_8888_init, \
-    pixman_composite_over_8888_8888_8888_cleanup, \
-    pixman_composite_over_8888_n_8888_process_pixblock_head, \
-    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
-    pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
-    28, /* dst_w_basereg */ \
-    4,  /* dst_r_basereg */ \
-    0,  /* src_basereg   */ \
-    12  /* mask_basereg  */
-
-generate_composite_function_single_scanline \
-    pixman_composite_scanline_over_mask_asm_neon, 32, 32, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    pixman_composite_over_8888_8888_8888_init, \
-    pixman_composite_over_8888_8888_8888_cleanup, \
-    pixman_composite_over_8888_n_8888_process_pixblock_head, \
-    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
-    pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
-    28, /* dst_w_basereg */ \
-    4,  /* dst_r_basereg */ \
-    0,  /* src_basereg   */ \
-    12  /* mask_basereg  */
-
-/******************************************************************************/
-
-/* TODO: expand macros and do better instructions scheduling */
-.macro pixman_composite_over_8888_8_8888_process_pixblock_tail_head
-    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
-    pixman_composite_over_8888_n_8888_process_pixblock_tail
-    vld4.8     {d0, d1, d2, d3}, [SRC]!
-    cache_preload 8, 8
-    vld1.8     {d15}, [MASK]!
-    pixman_composite_over_8888_n_8888_process_pixblock_head
-    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
-.endm
-
-.macro pixman_composite_over_8888_8_8888_init
-    vpush       {d8-d15}
-.endm
-
-.macro pixman_composite_over_8888_8_8888_cleanup
-    vpop        {d8-d15}
-.endm
-
-generate_composite_function \
-    pixman_composite_over_8888_8_8888_asm_neon, 32, 8, 32, \
-    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    5, /* prefetch distance */ \
-    pixman_composite_over_8888_8_8888_init, \
-    pixman_composite_over_8888_8_8888_cleanup, \
-    pixman_composite_over_8888_n_8888_process_pixblock_head, \
-    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
-    pixman_composite_over_8888_8_8888_process_pixblock_tail_head \
-    28, /* dst_w_basereg */ \
-    4,  /* dst_r_basereg */ \
-    0,  /* src_basereg   */ \
-    15  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_0888_0888_process_pixblock_head
-.endm
-
-.macro pixman_composite_src_0888_0888_process_pixblock_tail
-.endm
-
-.macro pixman_composite_src_0888_0888_process_pixblock_tail_head
-    vst3.8 {d0, d1, d2}, [DST_W]!
-    vld3.8 {d0, d1, d2}, [SRC]!
-    cache_preload 8, 8
-.endm
-
-generate_composite_function \
-    pixman_composite_src_0888_0888_asm_neon, 24, 0, 24, \
-    FLAG_DST_WRITEONLY, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_src_0888_0888_process_pixblock_head, \
-    pixman_composite_src_0888_0888_process_pixblock_tail, \
-    pixman_composite_src_0888_0888_process_pixblock_tail_head, \
-    0, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_0888_8888_rev_process_pixblock_head
-    vswp   d0, d2
-.endm
-
-.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail
-.endm
-
-.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail_head
-    vst4.8 {d0, d1, d2, d3}, [DST_W]!
-    vld3.8 {d0, d1, d2}, [SRC]!
-    vswp   d0, d2
-    cache_preload 8, 8
-.endm
-
-.macro pixman_composite_src_0888_8888_rev_init
-    veor   d3, d3, d3
-.endm
-
-generate_composite_function \
-    pixman_composite_src_0888_8888_rev_asm_neon, 24, 0, 32, \
-    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    pixman_composite_src_0888_8888_rev_init, \
-    default_cleanup, \
-    pixman_composite_src_0888_8888_rev_process_pixblock_head, \
-    pixman_composite_src_0888_8888_rev_process_pixblock_tail, \
-    pixman_composite_src_0888_8888_rev_process_pixblock_tail_head, \
-    0, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_0888_0565_rev_process_pixblock_head
-    vshll.u8    q8, d1, #8
-    vshll.u8    q9, d2, #8
-.endm
-
-.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail
-    vshll.u8    q14, d0, #8
-    vsri.u16    q14, q8, #5
-    vsri.u16    q14, q9, #11
-.endm
-
-.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail_head
-        vshll.u8    q14, d0, #8
-    vld3.8 {d0, d1, d2}, [SRC]!
-        vsri.u16    q14, q8, #5
-        vsri.u16    q14, q9, #11
-    vshll.u8    q8, d1, #8
-        vst1.16 {d28, d29}, [DST_W, :128]!
-    vshll.u8    q9, d2, #8
-.endm
-
-generate_composite_function \
-    pixman_composite_src_0888_0565_rev_asm_neon, 24, 0, 16, \
-    FLAG_DST_WRITEONLY, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_src_0888_0565_rev_process_pixblock_head, \
-    pixman_composite_src_0888_0565_rev_process_pixblock_tail, \
-    pixman_composite_src_0888_0565_rev_process_pixblock_tail_head, \
-    28, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
-
-/******************************************************************************/
-
-.macro pixman_composite_src_pixbuf_8888_process_pixblock_head
-    vmull.u8    q8, d3, d0
-    vmull.u8    q9, d3, d1
-    vmull.u8    q10, d3, d2
-.endm
-
-.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail
-    vrshr.u16   q11, q8, #8
-    vswp        d3, d31
-    vrshr.u16   q12, q9, #8
-    vrshr.u16   q13, q10, #8
-    vraddhn.u16 d30, q11, q8
-    vraddhn.u16 d29, q12, q9
-    vraddhn.u16 d28, q13, q10
-.endm
-
-.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail_head
-        vrshr.u16   q11, q8, #8
-        vswp        d3, d31
-        vrshr.u16   q12, q9, #8
-        vrshr.u16   q13, q10, #8
-    vld4.8 {d0, d1, d2, d3}, [SRC]!
-        vraddhn.u16 d30, q11, q8
-                                    PF add PF_X, PF_X, #8
-                                    PF tst PF_CTL, #0xF
-                                    PF addne PF_X, PF_X, #8
-                                    PF subne PF_CTL, PF_CTL, #1
-        vraddhn.u16 d29, q12, q9
-        vraddhn.u16 d28, q13, q10
-    vmull.u8    q8, d3, d0
-    vmull.u8    q9, d3, d1
-    vmull.u8    q10, d3, d2
-        vst4.8 {d28, d29, d30, d31}, [DST_W, :128]!
-                                    PF cmp PF_X, ORIG_W
-                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
-                                    PF subge PF_X, PF_X, ORIG_W
-                                    PF subges PF_CTL, PF_CTL, #0x10
-                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
-.endm
-
-generate_composite_function \
-    pixman_composite_src_pixbuf_8888_asm_neon, 32, 0, 32, \
-    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
-    8, /* number of pixels, processed in a single block */ \
-    10, /* prefetch distance */ \
-    default_init, \
-    default_cleanup, \
-    pixman_composite_src_pixbuf_8888_process_pixblock_head, \
-    pixman_composite_src_pixbuf_8888_process_pixblock_tail, \
-    pixman_composite_src_pixbuf_8888_process_pixblock_tail_head, \
-    28, /* dst_w_basereg */ \
-    0, /* dst_r_basereg */ \
-    0, /* src_basereg   */ \
-    0  /* mask_basereg  */
+/*

+ * Copyright © 2009 Nokia Corporation

+ *

+ * Permission is hereby granted, free of charge, to any person obtaining a

+ * copy of this software and associated documentation files (the "Software"),

+ * to deal in the Software without restriction, including without limitation

+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,

+ * and/or sell copies of the Software, and to permit persons to whom the

+ * Software is furnished to do so, subject to the following conditions:

+ *

+ * The above copyright notice and this permission notice (including the next

+ * paragraph) shall be included in all copies or substantial portions of the

+ * Software.

+ *

+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL

+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING

+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER

+ * DEALINGS IN THE SOFTWARE.

+ *

+ * Author:  Siarhei Siamashka (siarhei.siamashka@nokia.com)

+ */

+

+/*

+ * This file contains implementations of NEON optimized pixel processing

+ * functions. There is no full and detailed tutorial, but some functions

+ * (those which are exposing some new or interesting features) are

+ * extensively commented and can be used as examples.

+ *

+ * You may want to have a look at the comments for following functions:

+ *  - pixman_composite_over_8888_0565_asm_neon

+ *  - pixman_composite_over_n_8_0565_asm_neon

+ */

+

+/* Prevent the stack from becoming executable for no reason... */

+#if defined(__linux__) && defined(__ELF__)

+.section .note.GNU-stack,"",%progbits

+#endif

+

+    .text

+    .fpu neon

+    .arch armv7a

+    .object_arch armv4

+    .eabi_attribute 10, 0 /* suppress Tag_FP_arch */

+    .eabi_attribute 12, 0 /* suppress Tag_Advanced_SIMD_arch */

+    .arm

+    .altmacro

+

+#include "pixman-arm-neon-asm.h"

+

+/* Global configuration options and preferences */

+

+/*

+ * The code can optionally make use of unaligned memory accesses to improve

+ * performance of handling leading/trailing pixels for each scanline.

+ * Configuration variable RESPECT_STRICT_ALIGNMENT can be set to 0 for

+ * example in linux if unaligned memory accesses are not configured to

+ * generate.exceptions.

+ */

+.set RESPECT_STRICT_ALIGNMENT, 1

+

+/*

+ * Set default prefetch type. There is a choice between the following options:

+ *

+ * PREFETCH_TYPE_NONE (may be useful for the ARM cores where PLD is set to work

+ * as NOP to workaround some HW bugs or for whatever other reason)

+ *

+ * PREFETCH_TYPE_SIMPLE (may be useful for simple single-issue ARM cores where

+ * advanced prefetch intruduces heavy overhead)

+ *

+ * PREFETCH_TYPE_ADVANCED (useful for superscalar cores such as ARM Cortex-A8

+ * which can run ARM and NEON instructions simultaneously so that extra ARM

+ * instructions do not add (many) extra cycles, but improve prefetch efficiency)

+ *

+ * Note: some types of function can't support advanced prefetch and fallback

+ *       to simple one (those which handle 24bpp pixels)

+ */

+.set PREFETCH_TYPE_DEFAULT, PREFETCH_TYPE_ADVANCED

+

+/* Prefetch distance in pixels for simple prefetch */

+.set PREFETCH_DISTANCE_SIMPLE, 64

+

+/*

+ * Implementation of pixman_composite_over_8888_0565_asm_neon

+ *

+ * This function takes a8r8g8b8 source buffer, r5g6b5 destination buffer and

+ * performs OVER compositing operation. Function fast_composite_over_8888_0565

+ * from pixman-fast-path.c does the same in C and can be used as a reference.

+ *

+ * First we need to have some NEON assembly code which can do the actual

+ * operation on the pixels and provide it to the template macro.

+ *

+ * Template macro quite conveniently takes care of emitting all the necessary

+ * code for memory reading and writing (including quite tricky cases of

+ * handling unaligned leading/trailing pixels), so we only need to deal with

+ * the data in NEON registers.

+ *

+ * NEON registers allocation in general is recommented to be the following:

+ * d0,  d1,  d2,  d3  - contain loaded source pixel data

+ * d4,  d5,  d6,  d7  - contain loaded destination pixels (if they are needed)

+ * d24, d25, d26, d27 - contain loading mask pixel data (if mask is used)

+ * d28, d29, d30, d31 - place for storing the result (destination pixels)

+ *

+ * As can be seen above, four 64-bit NEON registers are used for keeping

+ * intermediate pixel data and up to 8 pixels can be processed in one step

+ * for 32bpp formats (16 pixels for 16bpp, 32 pixels for 8bpp).

+ *

+ * This particular function uses the following registers allocation:

+ * d0,  d1,  d2,  d3  - contain loaded source pixel data

+ * d4,  d5            - contain loaded destination pixels (they are needed)

+ * d28, d29           - place for storing the result (destination pixels)

+ */

+

+/*

+ * Step one. We need to have some code to do some arithmetics on pixel data.

+ * This is implemented as a pair of macros: '*_head' and '*_tail'. When used

+ * back-to-back, they take pixel data from {d0, d1, d2, d3} and {d4, d5},

+ * perform all the needed calculations and write the result to {d28, d29}.

+ * The rationale for having two macros and not just one will be explained

+ * later. In practice, any single monolitic function which does the work can

+ * be split into two parts in any arbitrary way without affecting correctness.

+ *

+ * There is one special trick here too. Common template macro can optionally

+ * make our life a bit easier by doing R, G, B, A color components

+ * deinterleaving for 32bpp pixel formats (and this feature is used in

+ * 'pixman_composite_over_8888_0565_asm_neon' function). So it means that

+ * instead of having 8 packed pixels in {d0, d1, d2, d3} registers, we

+ * actually use d0 register for blue channel (a vector of eight 8-bit

+ * values), d1 register for green, d2 for red and d3 for alpha. This

+ * simple conversion can be also done with a few NEON instructions:

+ *

+ * Packed to planar conversion:

+ *  vuzp.8 d0, d1

+ *  vuzp.8 d2, d3

+ *  vuzp.8 d1, d3

+ *  vuzp.8 d0, d2

+ *

+ * Planar to packed conversion:

+ *  vzip.8 d0, d2

+ *  vzip.8 d1, d3

+ *  vzip.8 d2, d3

+ *  vzip.8 d0, d1

+ *

+ * But pixel can be loaded directly in planar format using VLD4.8 NEON

+ * instruction. It is 1 cycle slower than VLD1.32, so this is not always

+ * desirable, that's why deinterleaving is optional.

+ *

+ * But anyway, here is the code:

+ */

+.macro pixman_composite_over_8888_0565_process_pixblock_head

+    /* convert 8 r5g6b5 pixel data from {d4, d5} to planar 8-bit format

+       and put data into d6 - red, d7 - green, d30 - blue */

+    vshrn.u16   d6, q2, #8

+    vshrn.u16   d7, q2, #3

+    vsli.u16    q2, q2, #5

+    vsri.u8     d6, d6, #5

+    vmvn.8      d3, d3      /* invert source alpha */

+    vsri.u8     d7, d7, #6

+    vshrn.u16   d30, q2, #2

+    /* now do alpha blending, storing results in 8-bit planar format

+       into d16 - red, d19 - green, d18 - blue */

+    vmull.u8    q10, d3, d6

+    vmull.u8    q11, d3, d7

+    vmull.u8    q12, d3, d30

+    vrshr.u16   q13, q10, #8

+    vrshr.u16   q3, q11, #8

+    vrshr.u16   q15, q12, #8

+    vraddhn.u16 d20, q10, q13

+    vraddhn.u16 d23, q11, q3

+    vraddhn.u16 d22, q12, q15

+.endm

+

+.macro pixman_composite_over_8888_0565_process_pixblock_tail

+    /* ... continue alpha blending */

+    vqadd.u8    d16, d2, d20

+    vqadd.u8    q9, q0, q11

+    /* convert the result to r5g6b5 and store it into {d28, d29} */

+    vshll.u8    q14, d16, #8

+    vshll.u8    q8, d19, #8

+    vshll.u8    q9, d18, #8

+    vsri.u16    q14, q8, #5

+    vsri.u16    q14, q9, #11

+.endm

+

+/*

+ * OK, now we got almost everything that we need. Using the above two

+ * macros, the work can be done right. But now we want to optimize

+ * it a bit. ARM Cortex-A8 is an in-order core, and benefits really

+ * a lot from good code scheduling and software pipelining.

+ *

+ * Let's construct some code, which will run in the core main loop.

+ * Some pseudo-code of the main loop will look like this:

+ *   head

+ *   while (...) {

+ *     tail

+ *     head

+ *   }

+ *   tail

+ *

+ * It may look a bit weird, but this setup allows to hide instruction

+ * latencies better and also utilize dual-issue capability more

+ * efficiently (make pairs of load-store and ALU instructions).

+ *

+ * So what we need now is a '*_tail_head' macro, which will be used

+ * in the core main loop. A trivial straightforward implementation

+ * of this macro would look like this:

+ *

+ *   pixman_composite_over_8888_0565_process_pixblock_tail

+ *   vst1.16     {d28, d29}, [DST_W, :128]!

+ *   vld1.16     {d4, d5}, [DST_R, :128]!

+ *   vld4.32     {d0, d1, d2, d3}, [SRC]!

+ *   pixman_composite_over_8888_0565_process_pixblock_head

+ *   cache_preload 8, 8

+ *

+ * Now it also got some VLD/VST instructions. We simply can't move from

+ * processing one block of pixels to the other one with just arithmetics.

+ * The previously processed data needs to be written to memory and new

+ * data needs to be fetched. Fortunately, this main loop does not deal

+ * with partial leading/trailing pixels and can load/store a full block

+ * of pixels in a bulk. Additionally, destination buffer is already

+ * 16 bytes aligned here (which is good for performance).

+ *

+ * New things here are DST_R, DST_W, SRC and MASK identifiers. These

+ * are the aliases for ARM registers which are used as pointers for

+ * accessing data. We maintain separate pointers for reading and writing

+ * destination buffer (DST_R and DST_W).

+ *

+ * Another new thing is 'cache_preload' macro. It is used for prefetching

+ * data into CPU L2 cache and improve performance when dealing with large

+ * images which are far larger than cache size. It uses one argument

+ * (actually two, but they need to be the same here) - number of pixels

+ * in a block. Looking into 'pixman-arm-neon-asm.h' can provide some

+ * details about this macro. Moreover, if good performance is needed

+ * the code from this macro needs to be copied into '*_tail_head' macro

+ * and mixed with the rest of code for optimal instructions scheduling.

+ * We are actually doing it below.

+ *

+ * Now after all the explanations, here is the optimized code.

+ * Different instruction streams (originaling from '*_head', '*_tail'

+ * and 'cache_preload' macro) use different indentation levels for

+ * better readability. Actually taking the code from one of these

+ * indentation levels and ignoring a few VLD/VST instructions would

+ * result in exactly the code from '*_head', '*_tail' or 'cache_preload'

+ * macro!

+ */

+

+#if 1

+

+.macro pixman_composite_over_8888_0565_process_pixblock_tail_head

+        vqadd.u8    d16, d2, d20

+    vld1.16     {d4, d5}, [DST_R, :128]!

+        vqadd.u8    q9, q0, q11

+    vshrn.u16   d6, q2, #8

+    vld4.8      {d0, d1, d2, d3}, [SRC]!

+    vshrn.u16   d7, q2, #3

+    vsli.u16    q2, q2, #5

+        vshll.u8    q14, d16, #8

+                                    PF add PF_X, PF_X, #8

+        vshll.u8    q8, d19, #8

+                                    PF tst PF_CTL, #0xF

+    vsri.u8     d6, d6, #5

+                                    PF addne PF_X, PF_X, #8

+    vmvn.8      d3, d3

+                                    PF subne PF_CTL, PF_CTL, #1

+    vsri.u8     d7, d7, #6

+    vshrn.u16   d30, q2, #2

+    vmull.u8    q10, d3, d6

+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]

+    vmull.u8    q11, d3, d7

+    vmull.u8    q12, d3, d30

+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]

+        vsri.u16    q14, q8, #5

+                                    PF cmp PF_X, ORIG_W

+        vshll.u8    q9, d18, #8

+    vrshr.u16   q13, q10, #8

+                                    PF subge PF_X, PF_X, ORIG_W

+    vrshr.u16   q3, q11, #8

+    vrshr.u16   q15, q12, #8

+                                    PF subges PF_CTL, PF_CTL, #0x10

+        vsri.u16    q14, q9, #11

+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!

+    vraddhn.u16 d20, q10, q13

+    vraddhn.u16 d23, q11, q3

+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!

+    vraddhn.u16 d22, q12, q15

+        vst1.16     {d28, d29}, [DST_W, :128]!

+.endm

+

+#else

+

+/* If we did not care much about the performance, we would just use this... */

+.macro pixman_composite_over_8888_0565_process_pixblock_tail_head

+    pixman_composite_over_8888_0565_process_pixblock_tail

+    vst1.16     {d28, d29}, [DST_W, :128]!

+    vld1.16     {d4, d5}, [DST_R, :128]!

+    vld4.32     {d0, d1, d2, d3}, [SRC]!

+    pixman_composite_over_8888_0565_process_pixblock_head

+    cache_preload 8, 8

+.endm

+

+#endif

+

+/*

+ * And now the final part. We are using 'generate_composite_function' macro

+ * to put all the stuff together. We are specifying the name of the function

+ * which we want to get, number of bits per pixel for the source, mask and

+ * destination (0 if unused, like mask in this case). Next come some bit

+ * flags:

+ *   FLAG_DST_READWRITE      - tells that the destination buffer is both read

+ *                             and written, for write-only buffer we would use

+ *                             FLAG_DST_WRITEONLY flag instead

+ *   FLAG_DEINTERLEAVE_32BPP - tells that we prefer to work with planar data

+ *                             and separate color channels for 32bpp format.

+ * The next things are:

+ *  - the number of pixels processed per iteration (8 in this case, because

+ *    that's the maximum what can fit into four 64-bit NEON registers).

+ *  - prefetch distance, measured in pixel blocks. In this case it is 5 times

+ *    by 8 pixels. That would be 40 pixels, or up to 160 bytes. Optimal

+ *    prefetch distance can be selected by running some benchmarks.

+ *

+ * After that we specify some macros, these are 'default_init',

+ * 'default_cleanup' here which are empty (but it is possible to have custom

+ * init/cleanup macros to be able to save/restore some extra NEON registers

+ * like d8-d15 or do anything else) followed by

+ * 'pixman_composite_over_8888_0565_process_pixblock_head',

+ * 'pixman_composite_over_8888_0565_process_pixblock_tail' and

+ * 'pixman_composite_over_8888_0565_process_pixblock_tail_head'

+ * which we got implemented above.

+ *

+ * The last part is the NEON registers allocation scheme.

+ */

+generate_composite_function \

+    pixman_composite_over_8888_0565_asm_neon, 32, 0, 16, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_over_8888_0565_process_pixblock_head, \

+    pixman_composite_over_8888_0565_process_pixblock_tail, \

+    pixman_composite_over_8888_0565_process_pixblock_tail_head, \

+    28, /* dst_w_basereg */ \

+    4,  /* dst_r_basereg */ \

+    0,  /* src_basereg   */ \

+    24  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_over_n_0565_process_pixblock_head

+    /* convert 8 r5g6b5 pixel data from {d4, d5} to planar 8-bit format

+       and put data into d6 - red, d7 - green, d30 - blue */

+    vshrn.u16   d6, q2, #8

+    vshrn.u16   d7, q2, #3

+    vsli.u16    q2, q2, #5

+    vsri.u8     d6, d6, #5

+    vsri.u8     d7, d7, #6

+    vshrn.u16   d30, q2, #2

+    /* now do alpha blending, storing results in 8-bit planar format

+       into d16 - red, d19 - green, d18 - blue */

+    vmull.u8    q10, d3, d6

+    vmull.u8    q11, d3, d7

+    vmull.u8    q12, d3, d30

+    vrshr.u16   q13, q10, #8

+    vrshr.u16   q3, q11, #8

+    vrshr.u16   q15, q12, #8

+    vraddhn.u16 d20, q10, q13

+    vraddhn.u16 d23, q11, q3

+    vraddhn.u16 d22, q12, q15

+.endm

+

+.macro pixman_composite_over_n_0565_process_pixblock_tail

+    /* ... continue alpha blending */

+    vqadd.u8    d16, d2, d20

+    vqadd.u8    q9, q0, q11

+    /* convert the result to r5g6b5 and store it into {d28, d29} */

+    vshll.u8    q14, d16, #8

+    vshll.u8    q8, d19, #8

+    vshll.u8    q9, d18, #8

+    vsri.u16    q14, q8, #5

+    vsri.u16    q14, q9, #11

+.endm

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_over_n_0565_process_pixblock_tail_head

+    pixman_composite_over_n_0565_process_pixblock_tail

+    vld1.16     {d4, d5}, [DST_R, :128]!

+    vst1.16     {d28, d29}, [DST_W, :128]!

+    pixman_composite_over_n_0565_process_pixblock_head

+.endm

+

+.macro pixman_composite_over_n_0565_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vld1.32     {d3[0]}, [DUMMY]

+    vdup.8      d0, d3[0]

+    vdup.8      d1, d3[1]

+    vdup.8      d2, d3[2]

+    vdup.8      d3, d3[3]

+    vmvn.8      d3, d3      /* invert source alpha */

+.endm

+

+generate_composite_function \

+    pixman_composite_over_n_0565_asm_neon, 0, 0, 16, \

+    FLAG_DST_READWRITE, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_over_n_0565_init, \

+    default_cleanup, \

+    pixman_composite_over_n_0565_process_pixblock_head, \

+    pixman_composite_over_n_0565_process_pixblock_tail, \

+    pixman_composite_over_n_0565_process_pixblock_tail_head, \

+    28, /* dst_w_basereg */ \

+    4,  /* dst_r_basereg */ \

+    0,  /* src_basereg   */ \

+    24  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_8888_0565_process_pixblock_head

+    vshll.u8    q8, d1, #8

+    vshll.u8    q14, d2, #8

+    vshll.u8    q9, d0, #8

+.endm

+

+.macro pixman_composite_src_8888_0565_process_pixblock_tail

+    vsri.u16    q14, q8, #5

+    vsri.u16    q14, q9, #11

+.endm

+

+.macro pixman_composite_src_8888_0565_process_pixblock_tail_head

+        vsri.u16    q14, q8, #5

+                                    PF add PF_X, PF_X, #8

+                                    PF tst PF_CTL, #0xF

+    vld4.8      {d0, d1, d2, d3}, [SRC]!

+                                    PF addne PF_X, PF_X, #8

+                                    PF subne PF_CTL, PF_CTL, #1

+        vsri.u16    q14, q9, #11

+                                    PF cmp PF_X, ORIG_W

+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]

+    vshll.u8    q8, d1, #8

+        vst1.16     {d28, d29}, [DST_W, :128]!

+                                    PF subge PF_X, PF_X, ORIG_W

+                                    PF subges PF_CTL, PF_CTL, #0x10

+    vshll.u8    q14, d2, #8

+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!

+    vshll.u8    q9, d0, #8

+.endm

+

+generate_composite_function \

+    pixman_composite_src_8888_0565_asm_neon, 32, 0, 16, \

+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_src_8888_0565_process_pixblock_head, \

+    pixman_composite_src_8888_0565_process_pixblock_tail, \

+    pixman_composite_src_8888_0565_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_src_0565_8888_process_pixblock_head

+    vshrn.u16   d30, q0, #8

+    vshrn.u16   d29, q0, #3

+    vsli.u16    q0, q0, #5

+    vmov.u8     d31, #255

+    vsri.u8     d30, d30, #5

+    vsri.u8     d29, d29, #6

+    vshrn.u16   d28, q0, #2

+.endm

+

+.macro pixman_composite_src_0565_8888_process_pixblock_tail

+.endm

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_src_0565_8888_process_pixblock_tail_head

+    pixman_composite_src_0565_8888_process_pixblock_tail

+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!

+    vld1.16    {d0, d1}, [SRC]!

+    pixman_composite_src_0565_8888_process_pixblock_head

+    cache_preload 8, 8

+.endm

+

+generate_composite_function \

+    pixman_composite_src_0565_8888_asm_neon, 16, 0, 32, \

+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_src_0565_8888_process_pixblock_head, \

+    pixman_composite_src_0565_8888_process_pixblock_tail, \

+    pixman_composite_src_0565_8888_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_add_8000_8000_process_pixblock_head

+    vqadd.u8    q14, q0, q2

+    vqadd.u8    q15, q1, q3

+.endm

+

+.macro pixman_composite_add_8000_8000_process_pixblock_tail

+.endm

+

+.macro pixman_composite_add_8000_8000_process_pixblock_tail_head

+    vld1.8      {d0, d1, d2, d3}, [SRC]!

+                                    PF add PF_X, PF_X, #32

+                                    PF tst PF_CTL, #0xF

+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!

+                                    PF addne PF_X, PF_X, #32

+                                    PF subne PF_CTL, PF_CTL, #1

+        vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!

+                                    PF cmp PF_X, ORIG_W

+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]

+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]

+                                    PF subge PF_X, PF_X, ORIG_W

+                                    PF subges PF_CTL, PF_CTL, #0x10

+    vqadd.u8    q14, q0, q2

+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!

+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!

+    vqadd.u8    q15, q1, q3

+.endm

+

+generate_composite_function \

+    pixman_composite_add_8000_8000_asm_neon, 8, 0, 8, \

+    FLAG_DST_READWRITE, \

+    32, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_add_8000_8000_process_pixblock_head, \

+    pixman_composite_add_8000_8000_process_pixblock_tail, \

+    pixman_composite_add_8000_8000_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_add_8888_8888_process_pixblock_tail_head

+    vld1.8      {d0, d1, d2, d3}, [SRC]!

+                                    PF add PF_X, PF_X, #8

+                                    PF tst PF_CTL, #0xF

+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!

+                                    PF addne PF_X, PF_X, #8

+                                    PF subne PF_CTL, PF_CTL, #1

+        vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!

+                                    PF cmp PF_X, ORIG_W

+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]

+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]

+                                    PF subge PF_X, PF_X, ORIG_W

+                                    PF subges PF_CTL, PF_CTL, #0x10

+    vqadd.u8    q14, q0, q2

+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!

+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!

+    vqadd.u8    q15, q1, q3

+.endm

+

+generate_composite_function \

+    pixman_composite_add_8888_8888_asm_neon, 32, 0, 32, \

+    FLAG_DST_READWRITE, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_add_8000_8000_process_pixblock_head, \

+    pixman_composite_add_8000_8000_process_pixblock_tail, \

+    pixman_composite_add_8888_8888_process_pixblock_tail_head

+

+generate_composite_function_single_scanline \

+    pixman_composite_scanline_add_asm_neon, 32, 0, 32, \

+    FLAG_DST_READWRITE, \

+    8, /* number of pixels, processed in a single block */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_add_8000_8000_process_pixblock_head, \

+    pixman_composite_add_8000_8000_process_pixblock_tail, \

+    pixman_composite_add_8888_8888_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_out_reverse_8888_8888_process_pixblock_head

+    vmvn.8      d24, d3  /* get inverted alpha */

+    /* do alpha blending */

+    vmull.u8    q8, d24, d4

+    vmull.u8    q9, d24, d5

+    vmull.u8    q10, d24, d6

+    vmull.u8    q11, d24, d7

+.endm

+

+.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail

+    vrshr.u16   q14, q8, #8

+    vrshr.u16   q15, q9, #8

+    vrshr.u16   q12, q10, #8

+    vrshr.u16   q13, q11, #8

+    vraddhn.u16 d28, q14, q8

+    vraddhn.u16 d29, q15, q9

+    vraddhn.u16 d30, q12, q10

+    vraddhn.u16 d31, q13, q11

+.endm

+

+.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head

+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!

+        vrshr.u16   q14, q8, #8

+                                    PF add PF_X, PF_X, #8

+                                    PF tst PF_CTL, #0xF

+        vrshr.u16   q15, q9, #8

+        vrshr.u16   q12, q10, #8

+        vrshr.u16   q13, q11, #8

+                                    PF addne PF_X, PF_X, #8

+                                    PF subne PF_CTL, PF_CTL, #1

+        vraddhn.u16 d28, q14, q8

+        vraddhn.u16 d29, q15, q9

+                                    PF cmp PF_X, ORIG_W

+        vraddhn.u16 d30, q12, q10

+        vraddhn.u16 d31, q13, q11

+    vld4.8      {d0, d1, d2, d3}, [SRC]!

+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]

+    vmvn.8      d22, d3

+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]

+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!

+                                    PF subge PF_X, PF_X, ORIG_W

+    vmull.u8    q8, d22, d4

+                                    PF subges PF_CTL, PF_CTL, #0x10

+    vmull.u8    q9, d22, d5

+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!

+    vmull.u8    q10, d22, d6

+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!

+    vmull.u8    q11, d22, d7

+.endm

+

+generate_composite_function_single_scanline \

+    pixman_composite_scanline_out_reverse_asm_neon, 32, 0, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_out_reverse_8888_8888_process_pixblock_head, \

+    pixman_composite_out_reverse_8888_8888_process_pixblock_tail, \

+    pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_over_8888_8888_process_pixblock_head

+    pixman_composite_out_reverse_8888_8888_process_pixblock_head

+.endm

+

+.macro pixman_composite_over_8888_8888_process_pixblock_tail

+    pixman_composite_out_reverse_8888_8888_process_pixblock_tail

+    vqadd.u8    q14, q0, q14

+    vqadd.u8    q15, q1, q15

+.endm

+

+.macro pixman_composite_over_8888_8888_process_pixblock_tail_head

+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!

+        vrshr.u16   q14, q8, #8

+                                    PF add PF_X, PF_X, #8

+                                    PF tst PF_CTL, #0xF

+        vrshr.u16   q15, q9, #8

+        vrshr.u16   q12, q10, #8

+        vrshr.u16   q13, q11, #8

+                                    PF addne PF_X, PF_X, #8

+                                    PF subne PF_CTL, PF_CTL, #1

+        vraddhn.u16 d28, q14, q8

+        vraddhn.u16 d29, q15, q9

+                                    PF cmp PF_X, ORIG_W

+        vraddhn.u16 d30, q12, q10

+        vraddhn.u16 d31, q13, q11

+        vqadd.u8    q14, q0, q14

+        vqadd.u8    q15, q1, q15

+    vld4.8      {d0, d1, d2, d3}, [SRC]!

+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]

+    vmvn.8      d22, d3

+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]

+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!

+                                    PF subge PF_X, PF_X, ORIG_W

+    vmull.u8    q8, d22, d4

+                                    PF subges PF_CTL, PF_CTL, #0x10

+    vmull.u8    q9, d22, d5

+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!

+    vmull.u8    q10, d22, d6

+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!

+    vmull.u8    q11, d22, d7

+.endm

+

+generate_composite_function \

+    pixman_composite_over_8888_8888_asm_neon, 32, 0, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_over_8888_8888_process_pixblock_head, \

+    pixman_composite_over_8888_8888_process_pixblock_tail, \

+    pixman_composite_over_8888_8888_process_pixblock_tail_head

+

+generate_composite_function_single_scanline \

+    pixman_composite_scanline_over_asm_neon, 32, 0, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_over_8888_8888_process_pixblock_head, \

+    pixman_composite_over_8888_8888_process_pixblock_tail, \

+    pixman_composite_over_8888_8888_process_pixblock_tail_head

+

+/******************************************************************************/

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_over_n_8888_process_pixblock_tail_head

+    pixman_composite_over_8888_8888_process_pixblock_tail

+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!

+    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!

+    pixman_composite_over_8888_8888_process_pixblock_head

+.endm

+

+.macro pixman_composite_over_n_8888_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vld1.32     {d3[0]}, [DUMMY]

+    vdup.8      d0, d3[0]

+    vdup.8      d1, d3[1]

+    vdup.8      d2, d3[2]

+    vdup.8      d3, d3[3]

+.endm

+

+generate_composite_function \

+    pixman_composite_over_n_8888_asm_neon, 0, 0, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_over_n_8888_init, \

+    default_cleanup, \

+    pixman_composite_over_8888_8888_process_pixblock_head, \

+    pixman_composite_over_8888_8888_process_pixblock_tail, \

+    pixman_composite_over_n_8888_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_over_reverse_n_8888_process_pixblock_tail_head

+        vrshr.u16   q14, q8, #8

+                                    PF add PF_X, PF_X, #8

+                                    PF tst PF_CTL, #0xF

+        vrshr.u16   q15, q9, #8

+        vrshr.u16   q12, q10, #8

+        vrshr.u16   q13, q11, #8

+                                    PF addne PF_X, PF_X, #8

+                                    PF subne PF_CTL, PF_CTL, #1

+        vraddhn.u16 d28, q14, q8

+        vraddhn.u16 d29, q15, q9

+                                    PF cmp PF_X, ORIG_W

+        vraddhn.u16 d30, q12, q10

+        vraddhn.u16 d31, q13, q11

+        vqadd.u8    q14, q0, q14

+        vqadd.u8    q15, q1, q15

+    vld4.8      {d0, d1, d2, d3}, [DST_R, :128]!

+    vmvn.8      d22, d3

+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]

+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!

+                                    PF subge PF_X, PF_X, ORIG_W

+    vmull.u8    q8, d22, d4

+                                    PF subges PF_CTL, PF_CTL, #0x10

+    vmull.u8    q9, d22, d5

+    vmull.u8    q10, d22, d6

+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!

+    vmull.u8    q11, d22, d7

+.endm

+

+.macro pixman_composite_over_reverse_n_8888_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vld1.32     {d7[0]}, [DUMMY]

+    vdup.8      d4, d7[0]

+    vdup.8      d5, d7[1]

+    vdup.8      d6, d7[2]

+    vdup.8      d7, d7[3]

+.endm

+

+generate_composite_function \

+    pixman_composite_over_reverse_n_8888_asm_neon, 0, 0, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_over_reverse_n_8888_init, \

+    default_cleanup, \

+    pixman_composite_over_8888_8888_process_pixblock_head, \

+    pixman_composite_over_8888_8888_process_pixblock_tail, \

+    pixman_composite_over_reverse_n_8888_process_pixblock_tail_head, \

+    28, /* dst_w_basereg */ \

+    0,  /* dst_r_basereg */ \

+    4,  /* src_basereg   */ \

+    24  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_over_n_8_0565_process_pixblock_head

+    /* in */

+    vmull.u8    q0, d24, d8

+    vmull.u8    q1, d24, d9

+    vmull.u8    q6, d24, d10

+    vmull.u8    q7, d24, d11

+    vrshr.u16   q10, q0, #8

+    vrshr.u16   q11, q1, #8

+    vrshr.u16   q12, q6, #8

+    vrshr.u16   q13, q7, #8

+    vraddhn.u16 d0, q0, q10

+    vraddhn.u16 d1, q1, q11

+    vraddhn.u16 d2, q6, q12

+    vraddhn.u16 d3, q7, q13

+

+    vshrn.u16   d6, q2, #8

+    vshrn.u16   d7, q2, #3

+    vsli.u16    q2, q2, #5

+    vsri.u8     d6, d6, #5

+    vmvn.8      d3, d3

+    vsri.u8     d7, d7, #6

+    vshrn.u16   d30, q2, #2

+    /* now do alpha blending */

+    vmull.u8    q10, d3, d6

+    vmull.u8    q11, d3, d7

+    vmull.u8    q12, d3, d30

+    vrshr.u16   q13, q10, #8

+    vrshr.u16   q3, q11, #8

+    vrshr.u16   q15, q12, #8

+    vraddhn.u16 d20, q10, q13

+    vraddhn.u16 d23, q11, q3

+    vraddhn.u16 d22, q12, q15

+.endm

+

+.macro pixman_composite_over_n_8_0565_process_pixblock_tail

+    vqadd.u8    d16, d2, d20

+    vqadd.u8    q9, q0, q11

+    /* convert to r5g6b5 */

+    vshll.u8    q14, d16, #8

+    vshll.u8    q8, d19, #8

+    vshll.u8    q9, d18, #8

+    vsri.u16    q14, q8, #5

+    vsri.u16    q14, q9, #11

+.endm

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_over_n_8_0565_process_pixblock_tail_head

+    pixman_composite_over_n_8_0565_process_pixblock_tail

+    vst1.16     {d28, d29}, [DST_W, :128]!

+    vld1.16     {d4, d5}, [DST_R, :128]!

+    vld1.8      {d24}, [MASK]!

+    cache_preload 8, 8

+    pixman_composite_over_n_8_0565_process_pixblock_head

+.endm

+

+/*

+ * This function needs a special initialization of solid mask.

+ * Solid source pixel data is fetched from stack at ARGS_STACK_OFFSET

+ * offset, split into color components and replicated in d8-d11

+ * registers. Additionally, this function needs all the NEON registers,

+ * so it has to save d8-d15 registers which are callee saved according

+ * to ABI. These registers are restored from 'cleanup' macro. All the

+ * other NEON registers are caller saved, so can be clobbered freely

+ * without introducing any problems.

+ */

+.macro pixman_composite_over_n_8_0565_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vpush       {d8-d15}

+    vld1.32     {d11[0]}, [DUMMY]

+    vdup.8      d8, d11[0]

+    vdup.8      d9, d11[1]

+    vdup.8      d10, d11[2]

+    vdup.8      d11, d11[3]

+.endm

+

+.macro pixman_composite_over_n_8_0565_cleanup

+    vpop        {d8-d15}

+.endm

+

+generate_composite_function \

+    pixman_composite_over_n_8_0565_asm_neon, 0, 8, 16, \

+    FLAG_DST_READWRITE, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_over_n_8_0565_init, \

+    pixman_composite_over_n_8_0565_cleanup, \

+    pixman_composite_over_n_8_0565_process_pixblock_head, \

+    pixman_composite_over_n_8_0565_process_pixblock_tail, \

+    pixman_composite_over_n_8_0565_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_src_0565_0565_process_pixblock_head

+.endm

+

+.macro pixman_composite_src_0565_0565_process_pixblock_tail

+.endm

+

+.macro pixman_composite_src_0565_0565_process_pixblock_tail_head

+    vst1.16 {d0, d1, d2, d3}, [DST_W, :128]!

+    vld1.16 {d0, d1, d2, d3}, [SRC]!

+    cache_preload 16, 16

+.endm

+

+generate_composite_function \

+    pixman_composite_src_0565_0565_asm_neon, 16, 0, 16, \

+    FLAG_DST_WRITEONLY, \

+    16, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_src_0565_0565_process_pixblock_head, \

+    pixman_composite_src_0565_0565_process_pixblock_tail, \

+    pixman_composite_src_0565_0565_process_pixblock_tail_head, \

+    0, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_n_8_process_pixblock_head

+.endm

+

+.macro pixman_composite_src_n_8_process_pixblock_tail

+.endm

+

+.macro pixman_composite_src_n_8_process_pixblock_tail_head

+    vst1.8  {d0, d1, d2, d3}, [DST_W, :128]!

+.endm

+

+.macro pixman_composite_src_n_8_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vld1.32     {d0[0]}, [DUMMY]

+    vsli.u64    d0, d0, #8

+    vsli.u64    d0, d0, #16

+    vsli.u64    d0, d0, #32

+    vorr        d1, d0, d0

+    vorr        q1, q0, q0

+.endm

+

+.macro pixman_composite_src_n_8_cleanup

+.endm

+

+generate_composite_function \

+    pixman_composite_src_n_8_asm_neon, 0, 0, 8, \

+    FLAG_DST_WRITEONLY, \

+    32, /* number of pixels, processed in a single block */ \

+    0,  /* prefetch distance */ \

+    pixman_composite_src_n_8_init, \

+    pixman_composite_src_n_8_cleanup, \

+    pixman_composite_src_n_8_process_pixblock_head, \

+    pixman_composite_src_n_8_process_pixblock_tail, \

+    pixman_composite_src_n_8_process_pixblock_tail_head, \

+    0, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_n_0565_process_pixblock_head

+.endm

+

+.macro pixman_composite_src_n_0565_process_pixblock_tail

+.endm

+

+.macro pixman_composite_src_n_0565_process_pixblock_tail_head

+    vst1.16 {d0, d1, d2, d3}, [DST_W, :128]!

+.endm

+

+.macro pixman_composite_src_n_0565_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vld1.32     {d0[0]}, [DUMMY]

+    vsli.u64    d0, d0, #16

+    vsli.u64    d0, d0, #32

+    vorr        d1, d0, d0

+    vorr        q1, q0, q0

+.endm

+

+.macro pixman_composite_src_n_0565_cleanup

+.endm

+

+generate_composite_function \

+    pixman_composite_src_n_0565_asm_neon, 0, 0, 16, \

+    FLAG_DST_WRITEONLY, \

+    16, /* number of pixels, processed in a single block */ \

+    0,  /* prefetch distance */ \

+    pixman_composite_src_n_0565_init, \

+    pixman_composite_src_n_0565_cleanup, \

+    pixman_composite_src_n_0565_process_pixblock_head, \

+    pixman_composite_src_n_0565_process_pixblock_tail, \

+    pixman_composite_src_n_0565_process_pixblock_tail_head, \

+    0, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_n_8888_process_pixblock_head

+.endm

+

+.macro pixman_composite_src_n_8888_process_pixblock_tail

+.endm

+

+.macro pixman_composite_src_n_8888_process_pixblock_tail_head

+    vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!

+.endm

+

+.macro pixman_composite_src_n_8888_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vld1.32     {d0[0]}, [DUMMY]

+    vsli.u64    d0, d0, #32

+    vorr        d1, d0, d0

+    vorr        q1, q0, q0

+.endm

+

+.macro pixman_composite_src_n_8888_cleanup

+.endm

+

+generate_composite_function \

+    pixman_composite_src_n_8888_asm_neon, 0, 0, 32, \

+    FLAG_DST_WRITEONLY, \

+    8, /* number of pixels, processed in a single block */ \

+    0, /* prefetch distance */ \

+    pixman_composite_src_n_8888_init, \

+    pixman_composite_src_n_8888_cleanup, \

+    pixman_composite_src_n_8888_process_pixblock_head, \

+    pixman_composite_src_n_8888_process_pixblock_tail, \

+    pixman_composite_src_n_8888_process_pixblock_tail_head, \

+    0, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_8888_8888_process_pixblock_head

+.endm

+

+.macro pixman_composite_src_8888_8888_process_pixblock_tail

+.endm

+

+.macro pixman_composite_src_8888_8888_process_pixblock_tail_head

+    vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!

+    vld1.32 {d0, d1, d2, d3}, [SRC]!

+    cache_preload 8, 8

+.endm

+

+generate_composite_function \

+    pixman_composite_src_8888_8888_asm_neon, 32, 0, 32, \

+    FLAG_DST_WRITEONLY, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_src_8888_8888_process_pixblock_head, \

+    pixman_composite_src_8888_8888_process_pixblock_tail, \

+    pixman_composite_src_8888_8888_process_pixblock_tail_head, \

+    0, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_x888_8888_process_pixblock_head

+    vorr     q0, q0, q2

+    vorr     q1, q1, q2

+.endm

+

+.macro pixman_composite_src_x888_8888_process_pixblock_tail

+.endm

+

+.macro pixman_composite_src_x888_8888_process_pixblock_tail_head

+    vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!

+    vld1.32 {d0, d1, d2, d3}, [SRC]!

+    vorr     q0, q0, q2

+    vorr     q1, q1, q2

+    cache_preload 8, 8

+.endm

+

+.macro pixman_composite_src_x888_8888_init

+    vmov.u8  q2, #0xFF

+    vshl.u32 q2, q2, #24

+.endm

+

+generate_composite_function \

+    pixman_composite_src_x888_8888_asm_neon, 32, 0, 32, \

+    FLAG_DST_WRITEONLY, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    pixman_composite_src_x888_8888_init, \

+    default_cleanup, \

+    pixman_composite_src_x888_8888_process_pixblock_head, \

+    pixman_composite_src_x888_8888_process_pixblock_tail, \

+    pixman_composite_src_x888_8888_process_pixblock_tail_head, \

+    0, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_over_n_8_8888_process_pixblock_head

+    /* expecting deinterleaved source data in {d8, d9, d10, d11} */

+    /* d8 - blue, d9 - green, d10 - red, d11 - alpha */

+    /* and destination data in {d4, d5, d6, d7} */

+    /* mask is in d24 (d25, d26, d27 are unused) */

+

+    /* in */

+    vmull.u8    q0, d24, d8

+    vmull.u8    q1, d24, d9

+    vmull.u8    q6, d24, d10

+    vmull.u8    q7, d24, d11

+    vrshr.u16   q10, q0, #8

+    vrshr.u16   q11, q1, #8

+    vrshr.u16   q12, q6, #8

+    vrshr.u16   q13, q7, #8

+    vraddhn.u16 d0, q0, q10

+    vraddhn.u16 d1, q1, q11

+    vraddhn.u16 d2, q6, q12

+    vraddhn.u16 d3, q7, q13

+    vmvn.8      d24, d3  /* get inverted alpha */

+    /* source:      d0 - blue, d1 - green, d2 - red, d3 - alpha */

+    /* destination: d4 - blue, d5 - green, d6 - red, d7 - alpha */

+    /* now do alpha blending */

+    vmull.u8    q8, d24, d4

+    vmull.u8    q9, d24, d5

+    vmull.u8    q10, d24, d6

+    vmull.u8    q11, d24, d7

+.endm

+

+.macro pixman_composite_over_n_8_8888_process_pixblock_tail

+    vrshr.u16   q14, q8, #8

+    vrshr.u16   q15, q9, #8

+    vrshr.u16   q12, q10, #8

+    vrshr.u16   q13, q11, #8

+    vraddhn.u16 d28, q14, q8

+    vraddhn.u16 d29, q15, q9

+    vraddhn.u16 d30, q12, q10

+    vraddhn.u16 d31, q13, q11

+    vqadd.u8    q14, q0, q14

+    vqadd.u8    q15, q1, q15

+.endm

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_over_n_8_8888_process_pixblock_tail_head

+    pixman_composite_over_n_8_8888_process_pixblock_tail

+    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!

+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!

+    vld1.8      {d24}, [MASK]!

+    cache_preload 8, 8

+    pixman_composite_over_n_8_8888_process_pixblock_head

+.endm

+

+.macro pixman_composite_over_n_8_8888_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vpush       {d8-d15}

+    vld1.32     {d11[0]}, [DUMMY]

+    vdup.8      d8, d11[0]

+    vdup.8      d9, d11[1]

+    vdup.8      d10, d11[2]

+    vdup.8      d11, d11[3]

+.endm

+

+.macro pixman_composite_over_n_8_8888_cleanup

+    vpop        {d8-d15}

+.endm

+

+generate_composite_function \

+    pixman_composite_over_n_8_8888_asm_neon, 0, 8, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_over_n_8_8888_init, \

+    pixman_composite_over_n_8_8888_cleanup, \

+    pixman_composite_over_n_8_8888_process_pixblock_head, \

+    pixman_composite_over_n_8_8888_process_pixblock_tail, \

+    pixman_composite_over_n_8_8888_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_head

+    /*

+     * 'combine_mask_ca' replacement

+     *

+     * input:  solid src (n) in {d8,  d9,  d10, d11}

+     *         dest in          {d4,  d5,  d6,  d7 }

+     *         mask in          {d24, d25, d26, d27}

+     * output: updated src in   {d0,  d1,  d2,  d3 }

+     *         updated mask in  {d24, d25, d26, d3 }

+     */

+    vmull.u8    q0,  d24, d8

+    vmull.u8    q1,  d25, d9

+    vmull.u8    q6,  d26, d10

+    vmull.u8    q7,  d27, d11

+    vmull.u8    q9,  d11, d25

+    vmull.u8    q12, d11, d24

+    vmull.u8    q13, d11, d26

+    vrshr.u16   q8,  q0,  #8

+    vrshr.u16   q10, q1,  #8

+    vrshr.u16   q11, q6,  #8

+    vraddhn.u16 d0,  q0,  q8

+    vraddhn.u16 d1,  q1,  q10

+    vraddhn.u16 d2,  q6,  q11

+    vrshr.u16   q11, q12, #8

+    vrshr.u16   q8,  q9,  #8

+    vrshr.u16   q6,  q13, #8

+    vrshr.u16   q10, q7,  #8

+    vraddhn.u16 d24, q12, q11

+    vraddhn.u16 d25, q9,  q8

+    vraddhn.u16 d26, q13, q6

+    vraddhn.u16 d3,  q7,  q10

+    /*

+     * 'combine_over_ca' replacement

+     *

+     * output: updated dest in {d28, d29, d30, d31}

+     */

+    vmvn.8      d24, d24

+    vmvn.8      d25, d25

+    vmull.u8    q8,  d24, d4

+    vmull.u8    q9,  d25, d5

+    vmvn.8      d26, d26

+    vmvn.8      d27, d3

+    vmull.u8    q10, d26, d6

+    vmull.u8    q11, d27, d7

+.endm

+

+.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail

+    /* ... continue 'combine_over_ca' replacement */

+    vrshr.u16   q14, q8,  #8

+    vrshr.u16   q15, q9,  #8

+    vrshr.u16   q6,  q10, #8

+    vrshr.u16   q7,  q11, #8

+    vraddhn.u16 d28, q14, q8

+    vraddhn.u16 d29, q15, q9

+    vraddhn.u16 d30, q6,  q10

+    vraddhn.u16 d31, q7,  q11

+    vqadd.u8    q14, q0,  q14

+    vqadd.u8    q15, q1,  q15

+.endm

+

+.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head

+        vrshr.u16   q14, q8, #8

+        vrshr.u16   q15, q9, #8

+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!

+        vrshr.u16   q6, q10, #8

+        vrshr.u16   q7, q11, #8

+        vraddhn.u16 d28, q14, q8

+        vraddhn.u16 d29, q15, q9

+        vraddhn.u16 d30, q6, q10

+        vraddhn.u16 d31, q7, q11

+    vld4.8      {d24, d25, d26, d27}, [MASK]!

+        vqadd.u8    q14, q0, q14

+        vqadd.u8    q15, q1, q15

+    cache_preload 8, 8

+    pixman_composite_over_n_8888_8888_ca_process_pixblock_head

+    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!

+.endm

+

+.macro pixman_composite_over_n_8888_8888_ca_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vpush       {d8-d15}

+    vld1.32     {d11[0]}, [DUMMY]

+    vdup.8      d8, d11[0]

+    vdup.8      d9, d11[1]

+    vdup.8      d10, d11[2]

+    vdup.8      d11, d11[3]

+.endm

+

+.macro pixman_composite_over_n_8888_8888_ca_cleanup

+    vpop        {d8-d15}

+.endm

+

+generate_composite_function \

+    pixman_composite_over_n_8888_8888_ca_asm_neon, 0, 32, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_over_n_8888_8888_ca_init, \

+    pixman_composite_over_n_8888_8888_ca_cleanup, \

+    pixman_composite_over_n_8888_8888_ca_process_pixblock_head, \

+    pixman_composite_over_n_8888_8888_ca_process_pixblock_tail, \

+    pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_add_n_8_8_process_pixblock_head

+    /* expecting source data in {d8, d9, d10, d11} */

+    /* d8 - blue, d9 - green, d10 - red, d11 - alpha */

+    /* and destination data in {d4, d5, d6, d7} */

+    /* mask is in d24, d25, d26, d27 */

+    vmull.u8    q0, d24, d11

+    vmull.u8    q1, d25, d11

+    vmull.u8    q6, d26, d11

+    vmull.u8    q7, d27, d11

+    vrshr.u16   q10, q0, #8

+    vrshr.u16   q11, q1, #8

+    vrshr.u16   q12, q6, #8

+    vrshr.u16   q13, q7, #8

+    vraddhn.u16 d0, q0, q10

+    vraddhn.u16 d1, q1, q11

+    vraddhn.u16 d2, q6, q12

+    vraddhn.u16 d3, q7, q13

+    vqadd.u8    q14, q0, q2

+    vqadd.u8    q15, q1, q3

+.endm

+

+.macro pixman_composite_add_n_8_8_process_pixblock_tail

+.endm

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_add_n_8_8_process_pixblock_tail_head

+    pixman_composite_add_n_8_8_process_pixblock_tail

+    vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!

+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!

+    vld1.8      {d24, d25, d26, d27}, [MASK]!

+    cache_preload 32, 32

+    pixman_composite_add_n_8_8_process_pixblock_head

+.endm

+

+.macro pixman_composite_add_n_8_8_init

+    add         DUMMY, sp, #ARGS_STACK_OFFSET

+    vpush       {d8-d15}

+    vld1.32     {d11[0]}, [DUMMY]

+    vdup.8      d11, d11[3]

+.endm

+

+.macro pixman_composite_add_n_8_8_cleanup

+    vpop        {d8-d15}

+.endm

+

+generate_composite_function \

+    pixman_composite_add_n_8_8_asm_neon, 0, 8, 8, \

+    FLAG_DST_READWRITE, \

+    32, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_add_n_8_8_init, \

+    pixman_composite_add_n_8_8_cleanup, \

+    pixman_composite_add_n_8_8_process_pixblock_head, \

+    pixman_composite_add_n_8_8_process_pixblock_tail, \

+    pixman_composite_add_n_8_8_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_add_8_8_8_process_pixblock_head

+    /* expecting source data in {d0, d1, d2, d3} */

+    /* destination data in {d4, d5, d6, d7} */

+    /* mask in {d24, d25, d26, d27} */

+    vmull.u8    q8, d24, d0

+    vmull.u8    q9, d25, d1

+    vmull.u8    q10, d26, d2

+    vmull.u8    q11, d27, d3

+    vrshr.u16   q0, q8, #8

+    vrshr.u16   q1, q9, #8

+    vrshr.u16   q12, q10, #8

+    vrshr.u16   q13, q11, #8

+    vraddhn.u16 d0, q0, q8

+    vraddhn.u16 d1, q1, q9

+    vraddhn.u16 d2, q12, q10

+    vraddhn.u16 d3, q13, q11

+    vqadd.u8    q14, q0, q2

+    vqadd.u8    q15, q1, q3

+.endm

+

+.macro pixman_composite_add_8_8_8_process_pixblock_tail

+.endm

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_add_8_8_8_process_pixblock_tail_head

+    pixman_composite_add_8_8_8_process_pixblock_tail

+    vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!

+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!

+    vld1.8      {d24, d25, d26, d27}, [MASK]!

+    vld1.8      {d0, d1, d2, d3}, [SRC]!

+    cache_preload 32, 32

+    pixman_composite_add_8_8_8_process_pixblock_head

+.endm

+

+.macro pixman_composite_add_8_8_8_init

+.endm

+

+.macro pixman_composite_add_8_8_8_cleanup

+.endm

+

+generate_composite_function \

+    pixman_composite_add_8_8_8_asm_neon, 8, 8, 8, \

+    FLAG_DST_READWRITE, \

+    32, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_add_8_8_8_init, \

+    pixman_composite_add_8_8_8_cleanup, \

+    pixman_composite_add_8_8_8_process_pixblock_head, \

+    pixman_composite_add_8_8_8_process_pixblock_tail, \

+    pixman_composite_add_8_8_8_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_add_8888_8888_8888_process_pixblock_head

+    /* expecting source data in {d0, d1, d2, d3} */

+    /* destination data in {d4, d5, d6, d7} */

+    /* mask in {d24, d25, d26, d27} */

+    vmull.u8    q8, d27, d0

+    vmull.u8    q9, d27, d1

+    vmull.u8    q10, d27, d2

+    vmull.u8    q11, d27, d3

+    vrshr.u16   q0, q8, #8

+    vrshr.u16   q1, q9, #8

+    vrshr.u16   q12, q10, #8

+    vrshr.u16   q13, q11, #8

+    vraddhn.u16 d0, q0, q8

+    vraddhn.u16 d1, q1, q9

+    vraddhn.u16 d2, q12, q10

+    vraddhn.u16 d3, q13, q11

+    vqadd.u8    q14, q0, q2

+    vqadd.u8    q15, q1, q3

+.endm

+

+.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail

+.endm

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail_head

+    pixman_composite_add_8888_8888_8888_process_pixblock_tail

+    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!

+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!

+    vld4.8      {d24, d25, d26, d27}, [MASK]!

+    vld4.8      {d0, d1, d2, d3}, [SRC]!

+    cache_preload 8, 8

+    pixman_composite_add_8888_8888_8888_process_pixblock_head

+.endm

+

+generate_composite_function \

+    pixman_composite_add_8888_8888_8888_asm_neon, 32, 32, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \

+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \

+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head

+

+generate_composite_function_single_scanline \

+    pixman_composite_scanline_add_mask_asm_neon, 32, 32, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \

+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \

+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head

+

+/******************************************************************************/

+

+.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_head

+    /* expecting source data in {d0, d1, d2, d3} */

+    /* destination data in {d4, d5, d6, d7} */

+    /* solid mask is in d15 */

+

+    /* 'in' */

+    vmull.u8    q8, d15, d3

+    vmull.u8    q6, d15, d2

+    vmull.u8    q5, d15, d1

+    vmull.u8    q4, d15, d0

+    vrshr.u16   q13, q8, #8

+    vrshr.u16   q12, q6, #8

+    vrshr.u16   q11, q5, #8

+    vrshr.u16   q10, q4, #8

+    vraddhn.u16 d3, q8, q13

+    vraddhn.u16 d2, q6, q12

+    vraddhn.u16 d1, q5, q11

+    vraddhn.u16 d0, q4, q10

+    vmvn.8      d24, d3  /* get inverted alpha */

+    /* now do alpha blending */

+    vmull.u8    q8, d24, d4

+    vmull.u8    q9, d24, d5

+    vmull.u8    q10, d24, d6

+    vmull.u8    q11, d24, d7

+.endm

+

+.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail

+    vrshr.u16   q14, q8, #8

+    vrshr.u16   q15, q9, #8

+    vrshr.u16   q12, q10, #8

+    vrshr.u16   q13, q11, #8

+    vraddhn.u16 d28, q14, q8

+    vraddhn.u16 d29, q15, q9

+    vraddhn.u16 d30, q12, q10

+    vraddhn.u16 d31, q13, q11

+.endm

+

+.macro pixman_composite_out_reverse_8888_8888_8888_init

+    vpush       {d8-d15}

+.endm

+

+.macro pixman_composite_out_reverse_8888_8888_8888_cleanup

+    vpop        {d8-d15}

+.endm

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head

+    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!

+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail

+    vld4.8     {d0, d1, d2, d3}, [SRC]!

+    cache_preload 8, 8

+    vld4.8     {d12, d13, d14, d15}, [MASK]!

+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_head

+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!

+.endm

+

+generate_composite_function_single_scanline \

+    pixman_composite_scanline_out_reverse_mask_asm_neon, 32, 32, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    pixman_composite_out_reverse_8888_8888_8888_init, \

+    pixman_composite_out_reverse_8888_8888_8888_cleanup, \

+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_head, \

+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail, \

+    pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head \

+    28, /* dst_w_basereg */ \

+    4,  /* dst_r_basereg */ \

+    0,  /* src_basereg   */ \

+    12  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_over_8888_n_8888_process_pixblock_head

+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_head

+.endm

+

+.macro pixman_composite_over_8888_n_8888_process_pixblock_tail

+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail

+    vqadd.u8    q14, q0, q14

+    vqadd.u8    q15, q1, q15

+.endm

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_over_8888_n_8888_process_pixblock_tail_head

+    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!

+    pixman_composite_over_8888_n_8888_process_pixblock_tail

+    vld4.8     {d0, d1, d2, d3}, [SRC]!

+    cache_preload 8, 8

+    pixman_composite_over_8888_n_8888_process_pixblock_head

+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!

+.endm

+

+.macro pixman_composite_over_8888_n_8888_init

+    add         DUMMY, sp, #48

+    vpush       {d8-d15}

+    vld1.32     {d15[0]}, [DUMMY]

+    vdup.8      d15, d15[3]

+.endm

+

+.macro pixman_composite_over_8888_n_8888_cleanup

+    vpop        {d8-d15}

+.endm

+

+generate_composite_function \

+    pixman_composite_over_8888_n_8888_asm_neon, 32, 0, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_over_8888_n_8888_init, \

+    pixman_composite_over_8888_n_8888_cleanup, \

+    pixman_composite_over_8888_n_8888_process_pixblock_head, \

+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \

+    pixman_composite_over_8888_n_8888_process_pixblock_tail_head

+

+/******************************************************************************/

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_over_8888_8888_8888_process_pixblock_tail_head

+    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!

+    pixman_composite_over_8888_n_8888_process_pixblock_tail

+    vld4.8     {d0, d1, d2, d3}, [SRC]!

+    cache_preload 8, 8

+    vld4.8     {d12, d13, d14, d15}, [MASK]!

+    pixman_composite_over_8888_n_8888_process_pixblock_head

+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!

+.endm

+

+.macro pixman_composite_over_8888_8888_8888_init

+    vpush       {d8-d15}

+.endm

+

+.macro pixman_composite_over_8888_8888_8888_cleanup

+    vpop        {d8-d15}

+.endm

+

+generate_composite_function \

+    pixman_composite_over_8888_8888_8888_asm_neon, 32, 32, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_over_8888_8888_8888_init, \

+    pixman_composite_over_8888_8888_8888_cleanup, \

+    pixman_composite_over_8888_n_8888_process_pixblock_head, \

+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \

+    pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \

+    28, /* dst_w_basereg */ \

+    4,  /* dst_r_basereg */ \

+    0,  /* src_basereg   */ \

+    12  /* mask_basereg  */

+

+generate_composite_function_single_scanline \

+    pixman_composite_scanline_over_mask_asm_neon, 32, 32, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    pixman_composite_over_8888_8888_8888_init, \

+    pixman_composite_over_8888_8888_8888_cleanup, \

+    pixman_composite_over_8888_n_8888_process_pixblock_head, \

+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \

+    pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \

+    28, /* dst_w_basereg */ \

+    4,  /* dst_r_basereg */ \

+    0,  /* src_basereg   */ \

+    12  /* mask_basereg  */

+

+/******************************************************************************/

+

+/* TODO: expand macros and do better instructions scheduling */

+.macro pixman_composite_over_8888_8_8888_process_pixblock_tail_head

+    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!

+    pixman_composite_over_8888_n_8888_process_pixblock_tail

+    vld4.8     {d0, d1, d2, d3}, [SRC]!

+    cache_preload 8, 8

+    vld1.8     {d15}, [MASK]!

+    pixman_composite_over_8888_n_8888_process_pixblock_head

+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!

+.endm

+

+.macro pixman_composite_over_8888_8_8888_init

+    vpush       {d8-d15}

+.endm

+

+.macro pixman_composite_over_8888_8_8888_cleanup

+    vpop        {d8-d15}

+.endm

+

+generate_composite_function \

+    pixman_composite_over_8888_8_8888_asm_neon, 32, 8, 32, \

+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    5, /* prefetch distance */ \

+    pixman_composite_over_8888_8_8888_init, \

+    pixman_composite_over_8888_8_8888_cleanup, \

+    pixman_composite_over_8888_n_8888_process_pixblock_head, \

+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \

+    pixman_composite_over_8888_8_8888_process_pixblock_tail_head \

+    28, /* dst_w_basereg */ \

+    4,  /* dst_r_basereg */ \

+    0,  /* src_basereg   */ \

+    15  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_0888_0888_process_pixblock_head

+.endm

+

+.macro pixman_composite_src_0888_0888_process_pixblock_tail

+.endm

+

+.macro pixman_composite_src_0888_0888_process_pixblock_tail_head

+    vst3.8 {d0, d1, d2}, [DST_W]!

+    vld3.8 {d0, d1, d2}, [SRC]!

+    cache_preload 8, 8

+.endm

+

+generate_composite_function \

+    pixman_composite_src_0888_0888_asm_neon, 24, 0, 24, \

+    FLAG_DST_WRITEONLY, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_src_0888_0888_process_pixblock_head, \

+    pixman_composite_src_0888_0888_process_pixblock_tail, \

+    pixman_composite_src_0888_0888_process_pixblock_tail_head, \

+    0, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_0888_8888_rev_process_pixblock_head

+    vswp   d0, d2

+.endm

+

+.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail

+.endm

+

+.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail_head

+    vst4.8 {d0, d1, d2, d3}, [DST_W]!

+    vld3.8 {d0, d1, d2}, [SRC]!

+    vswp   d0, d2

+    cache_preload 8, 8

+.endm

+

+.macro pixman_composite_src_0888_8888_rev_init

+    veor   d3, d3, d3

+.endm

+

+generate_composite_function \

+    pixman_composite_src_0888_8888_rev_asm_neon, 24, 0, 32, \

+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    pixman_composite_src_0888_8888_rev_init, \

+    default_cleanup, \

+    pixman_composite_src_0888_8888_rev_process_pixblock_head, \

+    pixman_composite_src_0888_8888_rev_process_pixblock_tail, \

+    pixman_composite_src_0888_8888_rev_process_pixblock_tail_head, \

+    0, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_0888_0565_rev_process_pixblock_head

+    vshll.u8    q8, d1, #8

+    vshll.u8    q9, d2, #8

+.endm

+

+.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail

+    vshll.u8    q14, d0, #8

+    vsri.u16    q14, q8, #5

+    vsri.u16    q14, q9, #11

+.endm

+

+.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail_head

+        vshll.u8    q14, d0, #8

+    vld3.8 {d0, d1, d2}, [SRC]!

+        vsri.u16    q14, q8, #5

+        vsri.u16    q14, q9, #11

+    vshll.u8    q8, d1, #8

+        vst1.16 {d28, d29}, [DST_W, :128]!

+    vshll.u8    q9, d2, #8

+.endm

+

+generate_composite_function \

+    pixman_composite_src_0888_0565_rev_asm_neon, 24, 0, 16, \

+    FLAG_DST_WRITEONLY, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_src_0888_0565_rev_process_pixblock_head, \

+    pixman_composite_src_0888_0565_rev_process_pixblock_tail, \

+    pixman_composite_src_0888_0565_rev_process_pixblock_tail_head, \

+    28, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

+

+/******************************************************************************/

+

+.macro pixman_composite_src_pixbuf_8888_process_pixblock_head

+    vmull.u8    q8, d3, d0

+    vmull.u8    q9, d3, d1

+    vmull.u8    q10, d3, d2

+.endm

+

+.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail

+    vrshr.u16   q11, q8, #8

+    vswp        d3, d31

+    vrshr.u16   q12, q9, #8

+    vrshr.u16   q13, q10, #8

+    vraddhn.u16 d30, q11, q8

+    vraddhn.u16 d29, q12, q9

+    vraddhn.u16 d28, q13, q10

+.endm

+

+.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail_head

+        vrshr.u16   q11, q8, #8

+        vswp        d3, d31

+        vrshr.u16   q12, q9, #8

+        vrshr.u16   q13, q10, #8

+    vld4.8 {d0, d1, d2, d3}, [SRC]!

+        vraddhn.u16 d30, q11, q8

+                                    PF add PF_X, PF_X, #8

+                                    PF tst PF_CTL, #0xF

+                                    PF addne PF_X, PF_X, #8

+                                    PF subne PF_CTL, PF_CTL, #1

+        vraddhn.u16 d29, q12, q9

+        vraddhn.u16 d28, q13, q10

+    vmull.u8    q8, d3, d0

+    vmull.u8    q9, d3, d1

+    vmull.u8    q10, d3, d2

+        vst4.8 {d28, d29, d30, d31}, [DST_W, :128]!

+                                    PF cmp PF_X, ORIG_W

+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]

+                                    PF subge PF_X, PF_X, ORIG_W

+                                    PF subges PF_CTL, PF_CTL, #0x10

+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!

+.endm

+

+generate_composite_function \

+    pixman_composite_src_pixbuf_8888_asm_neon, 32, 0, 32, \

+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \

+    8, /* number of pixels, processed in a single block */ \

+    10, /* prefetch distance */ \

+    default_init, \

+    default_cleanup, \

+    pixman_composite_src_pixbuf_8888_process_pixblock_head, \

+    pixman_composite_src_pixbuf_8888_process_pixblock_tail, \

+    pixman_composite_src_pixbuf_8888_process_pixblock_tail_head, \

+    28, /* dst_w_basereg */ \

+    0, /* dst_r_basereg */ \

+    0, /* src_basereg   */ \

+    0  /* mask_basereg  */

diff --git a/pixman/pixman/pixman-arm-neon.c b/pixman/pixman/pixman-arm-neon.c
index 6808b3658..ce049b280 100644
--- a/pixman/pixman/pixman-arm-neon.c
+++ b/pixman/pixman/pixman-arm-neon.c
@@ -1,352 +1,354 @@
-/*
- * Copyright © 2009 ARM Ltd, Movial Creative Technologies Oy
- *
- * Permission to use, copy, modify, distribute, and sell this software and its
- * documentation for any purpose is hereby granted without fee, provided that
- * the above copyright notice appear in all copies and that both that
- * copyright notice and this permission notice appear in supporting
- * documentation, and that the name of ARM Ltd not be used in
- * advertising or publicity pertaining to distribution of the software without
- * specific, written prior permission.  ARM Ltd makes no
- * representations about the suitability of this software for any purpose.  It
- * is provided "as is" without express or implied warranty.
- *
- * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
- * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
- * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
- * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
- * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
- * SOFTWARE.
- *
- * Author:  Ian Rickards (ian.rickards@arm.com)
- * Author:  Jonathan Morton (jonathan.morton@movial.com)
- * Author:  Markku Vire (markku.vire@movial.com)
- *
- */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <string.h>
-#include "pixman-private.h"
-#include "pixman-arm-common.h"
-
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_8888_8888,
-                                   uint32_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_x888_8888,
-                                   uint32_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0565_0565,
-                                   uint16_t, 1, uint16_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_0888,
-                                   uint8_t, 3, uint8_t, 3)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_8888_0565,
-                                   uint32_t, 1, uint16_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0565_8888,
-                                   uint16_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_8888_rev,
-                                   uint8_t, 3, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_0565_rev,
-                                   uint8_t, 3, uint16_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_pixbuf_8888,
-                                   uint32_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8000_8000,
-                                   uint8_t, 1, uint8_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8888_8888,
-                                   uint32_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_0565,
-                                   uint32_t, 1, uint16_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_8888,
-                                   uint32_t, 1, uint32_t, 1)
-
-PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_n_0565,
-                                 uint16_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_n_8888,
-                                 uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_reverse_n_8888,
-                                 uint32_t, 1)
-
-PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8_0565,
-                                      uint8_t, 1, uint16_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8_8888,
-                                      uint8_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8888_8888_ca,
-                                      uint32_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, add_n_8_8,
-                                      uint8_t, 1, uint8_t, 1)
-
-PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (neon, over_8888_n_8888,
-                                     uint32_t, 1, uint32_t, 1)
-
-PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8_8_8,
-                                        uint8_t, 1, uint8_t, 1, uint8_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8888_8888_8888,
-                                        uint32_t, 1, uint32_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8_8888,
-                                        uint32_t, 1, uint8_t, 1, uint32_t, 1)
-PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8888_8888,
-                                        uint32_t, 1, uint32_t, 1, uint32_t, 1)
-
-void
-pixman_composite_src_n_8_asm_neon (int32_t   w,
-                                   int32_t   h,
-                                   uint8_t  *dst,
-                                   int32_t   dst_stride,
-                                   uint8_t   src);
-
-void
-pixman_composite_src_n_0565_asm_neon (int32_t   w,
-                                      int32_t   h,
-                                      uint16_t *dst,
-                                      int32_t   dst_stride,
-                                      uint16_t  src);
-
-void
-pixman_composite_src_n_8888_asm_neon (int32_t   w,
-                                      int32_t   h,
-                                      uint32_t *dst,
-                                      int32_t   dst_stride,
-                                      uint32_t  src);
-
-static pixman_bool_t
-pixman_fill_neon (uint32_t *bits,
-                  int       stride,
-                  int       bpp,
-                  int       x,
-                  int       y,
-                  int       width,
-                  int       height,
-                  uint32_t  _xor)
-{
-    /* stride is always multiple of 32bit units in pixman */
-    uint32_t byte_stride = stride * sizeof(uint32_t);
-
-    switch (bpp)
-    {
-    case 8:
-	pixman_composite_src_n_8_asm_neon (
-		width,
-		height,
-		(uint8_t *)(((char *) bits) + y * byte_stride + x),
-		byte_stride,
-		_xor & 0xff);
-	return TRUE;
-    case 16:
-	pixman_composite_src_n_0565_asm_neon (
-		width,
-		height,
-		(uint16_t *)(((char *) bits) + y * byte_stride + x * 2),
-		byte_stride / 2,
-		_xor & 0xffff);
-	return TRUE;
-    case 32:
-	pixman_composite_src_n_8888_asm_neon (
-		width,
-		height,
-		(uint32_t *)(((char *) bits) + y * byte_stride + x * 4),
-		byte_stride / 4,
-		_xor);
-	return TRUE;
-    default:
-	return FALSE;
-    }
-}
-
-static pixman_bool_t
-pixman_blt_neon (uint32_t *src_bits,
-                 uint32_t *dst_bits,
-                 int       src_stride,
-                 int       dst_stride,
-                 int       src_bpp,
-                 int       dst_bpp,
-                 int       src_x,
-                 int       src_y,
-                 int       dst_x,
-                 int       dst_y,
-                 int       width,
-                 int       height)
-{
-    if (src_bpp != dst_bpp)
-	return FALSE;
-
-    switch (src_bpp)
-    {
-    case 16:
-	pixman_composite_src_0565_0565_asm_neon (
-		width, height,
-		(uint16_t *)(((char *) dst_bits) +
-		dst_y * dst_stride * 4 + dst_x * 2), dst_stride * 2,
-		(uint16_t *)(((char *) src_bits) +
-		src_y * src_stride * 4 + src_x * 2), src_stride * 2);
-	return TRUE;
-    case 32:
-	pixman_composite_src_8888_8888_asm_neon (
-		width, height,
-		(uint32_t *)(((char *) dst_bits) +
-		dst_y * dst_stride * 4 + dst_x * 4), dst_stride,
-		(uint32_t *)(((char *) src_bits) +
-		src_y * src_stride * 4 + src_x * 4), src_stride);
-	return TRUE;
-    default:
-	return FALSE;
-    }
-}
-
-static const pixman_fast_path_t arm_neon_fast_paths[] =
-{
-    PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     r5g6b5,   neon_composite_src_0565_0565),
-    PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     b5g6r5,   neon_composite_src_0565_0565),
-    PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     r5g6b5,   neon_composite_src_8888_0565),
-    PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     r5g6b5,   neon_composite_src_8888_0565),
-    PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     b5g6r5,   neon_composite_src_8888_0565),
-    PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     b5g6r5,   neon_composite_src_8888_0565),
-    PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     a8r8g8b8, neon_composite_src_0565_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     x8r8g8b8, neon_composite_src_0565_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     a8b8g8r8, neon_composite_src_0565_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     x8b8g8r8, neon_composite_src_0565_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     x8r8g8b8, neon_composite_src_8888_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     x8r8g8b8, neon_composite_src_8888_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     x8b8g8r8, neon_composite_src_8888_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     x8b8g8r8, neon_composite_src_8888_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     a8r8g8b8, neon_composite_src_8888_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     a8b8g8r8, neon_composite_src_8888_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     a8r8g8b8, neon_composite_src_x888_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     a8b8g8r8, neon_composite_src_x888_8888),
-    PIXMAN_STD_FAST_PATH (SRC,  r8g8b8,   null,     r8g8b8,   neon_composite_src_0888_0888),
-    PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     x8r8g8b8, neon_composite_src_0888_8888_rev),
-    PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     r5g6b5,   neon_composite_src_0888_0565_rev),
-    PIXMAN_STD_FAST_PATH (SRC,  pixbuf,   pixbuf,   a8r8g8b8, neon_composite_src_pixbuf_8888),
-    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       r5g6b5,   neon_composite_over_n_8_0565),
-    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       b5g6r5,   neon_composite_over_n_8_0565),
-    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8r8g8b8, neon_composite_over_n_8_8888),
-    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8r8g8b8, neon_composite_over_n_8_8888),
-    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8b8g8r8, neon_composite_over_n_8_8888),
-    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8b8g8r8, neon_composite_over_n_8_8888),
-    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     r5g6b5,   neon_composite_over_n_0565),
-    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     a8r8g8b8, neon_composite_over_n_8888),
-    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     x8r8g8b8, neon_composite_over_n_8888),
-    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, neon_composite_over_n_8888_8888_ca),
-    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, neon_composite_over_n_8888_8888_ca),
-    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, neon_composite_over_n_8888_8888_ca),
-    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, neon_composite_over_n_8888_8888_ca),
-    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    a8r8g8b8, neon_composite_over_8888_n_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    x8r8g8b8, neon_composite_over_8888_n_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       a8r8g8b8, neon_composite_over_8888_8_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       x8r8g8b8, neon_composite_over_8888_8_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       a8b8g8r8, neon_composite_over_8888_8_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       x8b8g8r8, neon_composite_over_8888_8_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_over_8888_8888_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     r5g6b5,   neon_composite_over_8888_0565),
-    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     b5g6r5,   neon_composite_over_8888_0565),
-    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     a8r8g8b8, neon_composite_over_8888_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     x8r8g8b8, neon_composite_over_8888_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     a8b8g8r8, neon_composite_over_8888_8888),
-    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     x8b8g8r8, neon_composite_over_8888_8888),
-    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null,     a8r8g8b8, neon_composite_src_x888_8888),
-    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null,     a8b8g8r8, neon_composite_src_x888_8888),
-    PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8,       neon_composite_add_n_8_8),
-    PIXMAN_STD_FAST_PATH (ADD,  a8,       a8,       a8,       neon_composite_add_8_8_8),
-    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_add_8888_8888_8888),
-    PIXMAN_STD_FAST_PATH (ADD,  a8,       null,     a8,       neon_composite_add_8000_8000),
-    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, null,     a8r8g8b8, neon_composite_add_8888_8888),
-    PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, null,     a8b8g8r8, neon_composite_add_8888_8888),
-    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, neon_composite_over_reverse_n_8888),
-    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, neon_composite_over_reverse_n_8888),
-
-    { PIXMAN_OP_NONE },
-};
-
-static pixman_bool_t
-arm_neon_blt (pixman_implementation_t *imp,
-              uint32_t *               src_bits,
-              uint32_t *               dst_bits,
-              int                      src_stride,
-              int                      dst_stride,
-              int                      src_bpp,
-              int                      dst_bpp,
-              int                      src_x,
-              int                      src_y,
-              int                      dst_x,
-              int                      dst_y,
-              int                      width,
-              int                      height)
-{
-    if (!pixman_blt_neon (
-            src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
-            src_x, src_y, dst_x, dst_y, width, height))
-
-    {
-	return _pixman_implementation_blt (
-	    imp->delegate,
-	    src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
-	    src_x, src_y, dst_x, dst_y, width, height);
-    }
-
-    return TRUE;
-}
-
-static pixman_bool_t
-arm_neon_fill (pixman_implementation_t *imp,
-               uint32_t *               bits,
-               int                      stride,
-               int                      bpp,
-               int                      x,
-               int                      y,
-               int                      width,
-               int                      height,
-               uint32_t xor)
-{
-    if (pixman_fill_neon (bits, stride, bpp, x, y, width, height, xor))
-	return TRUE;
-
-    return _pixman_implementation_fill (
-	imp->delegate, bits, stride, bpp, x, y, width, height, xor);
-}
-
-#define BIND_COMBINE_U(name)                                             \
-void                                                                     \
-pixman_composite_scanline_##name##_mask_asm_neon (int32_t         w,     \
-                                                  const uint32_t *dst,   \
-                                                  const uint32_t *src,   \
-                                                  const uint32_t *mask); \
-                                                                         \
-void                                                                     \
-pixman_composite_scanline_##name##_asm_neon (int32_t         w,          \
-                                             const uint32_t *dst,        \
-                                             const uint32_t *src);       \
-                                                                         \
-static void                                                              \
-neon_combine_##name##_u (pixman_implementation_t *imp,                   \
-                         pixman_op_t              op,                    \
-                         uint32_t *               dest,                  \
-                         const uint32_t *         src,                   \
-                         const uint32_t *         mask,                  \
-                         int                      width)                 \
-{                                                                        \
-    if (mask)                                                            \
-	pixman_composite_scanline_##name##_mask_asm_neon (width, dest,   \
-	                                                  src, mask);    \
-    else                                                                 \
-	pixman_composite_scanline_##name##_asm_neon (width, dest, src);  \
-}
-
-BIND_COMBINE_U (over)
-BIND_COMBINE_U (add)
-
-pixman_implementation_t *
-_pixman_implementation_create_arm_neon (void)
-{
-    pixman_implementation_t *general = _pixman_implementation_create_fast_path ();
-    pixman_implementation_t *imp =
-	_pixman_implementation_create (general, arm_neon_fast_paths);
-
-    imp->combine_32[PIXMAN_OP_OVER] = neon_combine_over_u;
-    imp->combine_32[PIXMAN_OP_ADD] = neon_combine_add_u;
-
-    imp->blt = arm_neon_blt;
-    imp->fill = arm_neon_fill;
-
-    return imp;
-}
+/*

+ * Copyright © 2009 ARM Ltd, Movial Creative Technologies Oy

+ *

+ * Permission to use, copy, modify, distribute, and sell this software and its

+ * documentation for any purpose is hereby granted without fee, provided that

+ * the above copyright notice appear in all copies and that both that

+ * copyright notice and this permission notice appear in supporting

+ * documentation, and that the name of ARM Ltd not be used in

+ * advertising or publicity pertaining to distribution of the software without

+ * specific, written prior permission.  ARM Ltd makes no

+ * representations about the suitability of this software for any purpose.  It

+ * is provided "as is" without express or implied warranty.

+ *

+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS

+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND

+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY

+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES

+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN

+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING

+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS

+ * SOFTWARE.

+ *

+ * Author:  Ian Rickards (ian.rickards@arm.com)

+ * Author:  Jonathan Morton (jonathan.morton@movial.com)

+ * Author:  Markku Vire (markku.vire@movial.com)

+ *

+ */

+

+#ifdef HAVE_CONFIG_H

+#include <config.h>

+#endif

+

+#include <string.h>

+#include "pixman-private.h"

+#include "pixman-arm-common.h"

+

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_8888_8888,

+                                   uint32_t, 1, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_x888_8888,

+                                   uint32_t, 1, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0565_0565,

+                                   uint16_t, 1, uint16_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_0888,

+                                   uint8_t, 3, uint8_t, 3)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_8888_0565,

+                                   uint32_t, 1, uint16_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0565_8888,

+                                   uint16_t, 1, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_8888_rev,

+                                   uint8_t, 3, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_0565_rev,

+                                   uint8_t, 3, uint16_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_pixbuf_8888,

+                                   uint32_t, 1, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8000_8000,

+                                   uint8_t, 1, uint8_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8888_8888,

+                                   uint32_t, 1, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_0565,

+                                   uint32_t, 1, uint16_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_8888,

+                                   uint32_t, 1, uint32_t, 1)

+

+PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_n_0565,

+                                 uint16_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_n_8888,

+                                 uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_reverse_n_8888,

+                                 uint32_t, 1)

+

+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8_0565,

+                                      uint8_t, 1, uint16_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8_8888,

+                                      uint8_t, 1, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8888_8888_ca,

+                                      uint32_t, 1, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, add_n_8_8,

+                                      uint8_t, 1, uint8_t, 1)

+

+PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (neon, over_8888_n_8888,

+                                     uint32_t, 1, uint32_t, 1)

+

+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8_8_8,

+                                        uint8_t, 1, uint8_t, 1, uint8_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8888_8888_8888,

+                                        uint32_t, 1, uint32_t, 1, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8_8888,

+                                        uint32_t, 1, uint8_t, 1, uint32_t, 1)

+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8888_8888,

+                                        uint32_t, 1, uint32_t, 1, uint32_t, 1)

+

+void

+pixman_composite_src_n_8_asm_neon (int32_t   w,

+                                   int32_t   h,

+                                   uint8_t  *dst,

+                                   int32_t   dst_stride,

+                                   uint8_t   src);

+

+void

+pixman_composite_src_n_0565_asm_neon (int32_t   w,

+                                      int32_t   h,

+                                      uint16_t *dst,

+                                      int32_t   dst_stride,

+                                      uint16_t  src);

+

+void

+pixman_composite_src_n_8888_asm_neon (int32_t   w,

+                                      int32_t   h,

+                                      uint32_t *dst,

+                                      int32_t   dst_stride,

+                                      uint32_t  src);

+

+static pixman_bool_t

+pixman_fill_neon (uint32_t *bits,

+                  int       stride,

+                  int       bpp,

+                  int       x,

+                  int       y,

+                  int       width,

+                  int       height,

+                  uint32_t  _xor)

+{

+    /* stride is always multiple of 32bit units in pixman */

+    uint32_t byte_stride = stride * sizeof(uint32_t);

+

+    switch (bpp)

+    {

+    case 8:

+	pixman_composite_src_n_8_asm_neon (

+		width,

+		height,

+		(uint8_t *)(((char *) bits) + y * byte_stride + x),

+		byte_stride,

+		_xor & 0xff);

+	return TRUE;

+    case 16:

+	pixman_composite_src_n_0565_asm_neon (

+		width,

+		height,

+		(uint16_t *)(((char *) bits) + y * byte_stride + x * 2),

+		byte_stride / 2,

+		_xor & 0xffff);

+	return TRUE;

+    case 32:

+	pixman_composite_src_n_8888_asm_neon (

+		width,

+		height,

+		(uint32_t *)(((char *) bits) + y * byte_stride + x * 4),

+		byte_stride / 4,

+		_xor);

+	return TRUE;

+    default:

+	return FALSE;

+    }

+}

+

+static pixman_bool_t

+pixman_blt_neon (uint32_t *src_bits,

+                 uint32_t *dst_bits,

+                 int       src_stride,

+                 int       dst_stride,

+                 int       src_bpp,

+                 int       dst_bpp,

+                 int       src_x,

+                 int       src_y,

+                 int       dst_x,

+                 int       dst_y,

+                 int       width,

+                 int       height)

+{

+    if (src_bpp != dst_bpp)

+	return FALSE;

+

+    switch (src_bpp)

+    {

+    case 16:

+	pixman_composite_src_0565_0565_asm_neon (

+		width, height,

+		(uint16_t *)(((char *) dst_bits) +

+		dst_y * dst_stride * 4 + dst_x * 2), dst_stride * 2,

+		(uint16_t *)(((char *) src_bits) +

+		src_y * src_stride * 4 + src_x * 2), src_stride * 2);

+	return TRUE;

+    case 32:

+	pixman_composite_src_8888_8888_asm_neon (

+		width, height,

+		(uint32_t *)(((char *) dst_bits) +

+		dst_y * dst_stride * 4 + dst_x * 4), dst_stride,

+		(uint32_t *)(((char *) src_bits) +

+		src_y * src_stride * 4 + src_x * 4), src_stride);

+	return TRUE;

+    default:

+	return FALSE;

+    }

+}

+

+static const pixman_fast_path_t arm_neon_fast_paths[] =

+{

+    PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     r5g6b5,   neon_composite_src_0565_0565),

+    PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     b5g6r5,   neon_composite_src_0565_0565),

+    PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     r5g6b5,   neon_composite_src_8888_0565),

+    PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     r5g6b5,   neon_composite_src_8888_0565),

+    PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     b5g6r5,   neon_composite_src_8888_0565),

+    PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     b5g6r5,   neon_composite_src_8888_0565),

+    PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     a8r8g8b8, neon_composite_src_0565_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     x8r8g8b8, neon_composite_src_0565_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     a8b8g8r8, neon_composite_src_0565_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     x8b8g8r8, neon_composite_src_0565_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     x8r8g8b8, neon_composite_src_8888_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     x8r8g8b8, neon_composite_src_8888_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     x8b8g8r8, neon_composite_src_8888_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     x8b8g8r8, neon_composite_src_8888_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     a8r8g8b8, neon_composite_src_8888_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     a8b8g8r8, neon_composite_src_8888_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     a8r8g8b8, neon_composite_src_x888_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     a8b8g8r8, neon_composite_src_x888_8888),

+    PIXMAN_STD_FAST_PATH (SRC,  r8g8b8,   null,     r8g8b8,   neon_composite_src_0888_0888),

+    PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     x8r8g8b8, neon_composite_src_0888_8888_rev),

+    PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     r5g6b5,   neon_composite_src_0888_0565_rev),

+    PIXMAN_STD_FAST_PATH (SRC,  pixbuf,   pixbuf,   a8r8g8b8, neon_composite_src_pixbuf_8888),

+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       r5g6b5,   neon_composite_over_n_8_0565),

+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       b5g6r5,   neon_composite_over_n_8_0565),

+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8r8g8b8, neon_composite_over_n_8_8888),

+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8r8g8b8, neon_composite_over_n_8_8888),

+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8b8g8r8, neon_composite_over_n_8_8888),

+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8b8g8r8, neon_composite_over_n_8_8888),

+    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     r5g6b5,   neon_composite_over_n_0565),

+    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     a8r8g8b8, neon_composite_over_n_8888),

+    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     x8r8g8b8, neon_composite_over_n_8888),

+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, neon_composite_over_n_8888_8888_ca),

+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, neon_composite_over_n_8888_8888_ca),

+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, neon_composite_over_n_8888_8888_ca),

+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, neon_composite_over_n_8888_8888_ca),

+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    a8r8g8b8, neon_composite_over_8888_n_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    x8r8g8b8, neon_composite_over_8888_n_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       a8r8g8b8, neon_composite_over_8888_8_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       x8r8g8b8, neon_composite_over_8888_8_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       a8b8g8r8, neon_composite_over_8888_8_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       x8b8g8r8, neon_composite_over_8888_8_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_over_8888_8888_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     r5g6b5,   neon_composite_over_8888_0565),

+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     b5g6r5,   neon_composite_over_8888_0565),

+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     a8r8g8b8, neon_composite_over_8888_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     x8r8g8b8, neon_composite_over_8888_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     a8b8g8r8, neon_composite_over_8888_8888),

+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     x8b8g8r8, neon_composite_over_8888_8888),

+    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null,     a8r8g8b8, neon_composite_src_x888_8888),

+    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null,     a8b8g8r8, neon_composite_src_x888_8888),

+    PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8,       neon_composite_add_n_8_8),

+    PIXMAN_STD_FAST_PATH (ADD,  a8,       a8,       a8,       neon_composite_add_8_8_8),

+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_add_8888_8888_8888),

+    PIXMAN_STD_FAST_PATH (ADD,  a8,       null,     a8,       neon_composite_add_8000_8000),

+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, null,     a8r8g8b8, neon_composite_add_8888_8888),

+    PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, null,     a8b8g8r8, neon_composite_add_8888_8888),

+    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, neon_composite_over_reverse_n_8888),

+    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, neon_composite_over_reverse_n_8888),

+

+    { PIXMAN_OP_NONE },

+};

+

+static pixman_bool_t

+arm_neon_blt (pixman_implementation_t *imp,

+              uint32_t *               src_bits,

+              uint32_t *               dst_bits,

+              int                      src_stride,

+              int                      dst_stride,

+              int                      src_bpp,

+              int                      dst_bpp,

+              int                      src_x,

+              int                      src_y,

+              int                      dst_x,

+              int                      dst_y,

+              int                      width,

+              int                      height)

+{

+    if (!pixman_blt_neon (

+            src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,

+            src_x, src_y, dst_x, dst_y, width, height))

+

+    {

+	return _pixman_implementation_blt (

+	    imp->delegate,

+	    src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,

+	    src_x, src_y, dst_x, dst_y, width, height);

+    }

+

+    return TRUE;

+}

+

+static pixman_bool_t

+arm_neon_fill (pixman_implementation_t *imp,

+               uint32_t *               bits,

+               int                      stride,

+               int                      bpp,

+               int                      x,

+               int                      y,

+               int                      width,

+               int                      height,

+               uint32_t xor)

+{

+    if (pixman_fill_neon (bits, stride, bpp, x, y, width, height, xor))

+	return TRUE;

+

+    return _pixman_implementation_fill (

+	imp->delegate, bits, stride, bpp, x, y, width, height, xor);

+}

+

+#define BIND_COMBINE_U(name)                                             \

+void                                                                     \

+pixman_composite_scanline_##name##_mask_asm_neon (int32_t         w,     \

+                                                  const uint32_t *dst,   \

+                                                  const uint32_t *src,   \

+                                                  const uint32_t *mask); \

+                                                                         \

+void                                                                     \

+pixman_composite_scanline_##name##_asm_neon (int32_t         w,          \

+                                             const uint32_t *dst,        \

+                                             const uint32_t *src);       \

+                                                                         \

+static void                                                              \

+neon_combine_##name##_u (pixman_implementation_t *imp,                   \

+                         pixman_op_t              op,                    \

+                         uint32_t *               dest,                  \

+                         const uint32_t *         src,                   \

+                         const uint32_t *         mask,                  \

+                         int                      width)                 \

+{                                                                        \

+    if (mask)                                                            \

+	pixman_composite_scanline_##name##_mask_asm_neon (width, dest,   \

+	                                                  src, mask);    \

+    else                                                                 \

+	pixman_composite_scanline_##name##_asm_neon (width, dest, src);  \

+}

+

+BIND_COMBINE_U (over)

+BIND_COMBINE_U (add)

+BIND_COMBINE_U (out_reverse)

+

+pixman_implementation_t *

+_pixman_implementation_create_arm_neon (void)

+{

+    pixman_implementation_t *general = _pixman_implementation_create_fast_path ();

+    pixman_implementation_t *imp =

+	_pixman_implementation_create (general, arm_neon_fast_paths);

+

+    imp->combine_32[PIXMAN_OP_OVER] = neon_combine_over_u;

+    imp->combine_32[PIXMAN_OP_ADD] = neon_combine_add_u;

+    imp->combine_32[PIXMAN_OP_OUT_REVERSE] = neon_combine_out_reverse_u;

+

+    imp->blt = arm_neon_blt;

+    imp->fill = arm_neon_fill;

+

+    return imp;

+}

diff --git a/pixman/pixman/pixman-fast-path.c b/pixman/pixman/pixman-fast-path.c
index 8165db4c7..6c214fede 100644
--- a/pixman/pixman/pixman-fast-path.c
+++ b/pixman/pixman/pixman-fast-path.c
@@ -1591,13 +1591,6 @@ fast_composite_scaled_nearest_ ## scale_func_name ## _ ## OP (pixman_implementat
 	if (w & 1)										\

 	{											\

 	    x1 = vx >> 16;									\

-	    vx += unit_x;									\

-	    if (PIXMAN_REPEAT_ ## repeat_mode == PIXMAN_REPEAT_NORMAL)				\

-	    {											\

-		/* This works because we know that unit_x is positive */			\

-		while (vx >= max_vx)								\

-		    vx -= max_vx;								\

-	    }											\

 	    s1 = src[x1];									\

 												\

 	    if (PIXMAN_OP_ ## OP == PIXMAN_OP_OVER)						\

diff --git a/pixman/pixman/pixman-image.c b/pixman/pixman/pixman-image.c
index 1cce9273d..3020f246c 100644
--- a/pixman/pixman/pixman-image.c
+++ b/pixman/pixman/pixman-image.c
@@ -101,6 +101,7 @@ _pixman_image_allocate (void)
 

 	pixman_region32_init (&common->clip_region);

 

+	common->alpha_count = 0;

 	common->have_clip_region = FALSE;

 	common->clip_sources = FALSE;

 	common->transform = NULL;

@@ -408,12 +409,14 @@ compute_image_info (pixman_image_t *image)
 	    }

 	}

 

-	if (image->common.repeat != PIXMAN_REPEAT_NONE				&&

-	    !PIXMAN_FORMAT_A (image->bits.format)				&&

+	if (!PIXMAN_FORMAT_A (image->bits.format)				&&

 	    PIXMAN_FORMAT_TYPE (image->bits.format) != PIXMAN_TYPE_GRAY		&&

 	    PIXMAN_FORMAT_TYPE (image->bits.format) != PIXMAN_TYPE_COLOR)

 	{

-	    flags |= FAST_PATH_IS_OPAQUE;

+	    flags |= FAST_PATH_SAMPLES_OPAQUE;

+

+	    if (image->common.repeat != PIXMAN_REPEAT_NONE)

+		flags |= FAST_PATH_IS_OPAQUE;

 	}

 

 	if (source_image_needs_out_of_bounds_workaround (&image->bits))

@@ -461,7 +464,7 @@ compute_image_info (pixman_image_t *image)
 	image->common.filter == PIXMAN_FILTER_CONVOLUTION	||

 	image->common.component_alpha)

     {

-	flags &= ~FAST_PATH_IS_OPAQUE;

+	flags &= ~(FAST_PATH_IS_OPAQUE | FAST_PATH_SAMPLES_OPAQUE);

     }

 

     image->common.flags = flags;

@@ -668,15 +671,41 @@ pixman_image_set_alpha_map (pixman_image_t *image,
 

     return_if_fail (!alpha_map || alpha_map->type == BITS);

 

+    if (alpha_map && common->alpha_count > 0)

+    {

+	/* If this image is being used as an alpha map itself,

+	 * then you can't give it an alpha map of its own.

+	 */

+	return;

+    }

+

+    if (alpha_map && alpha_map->common.alpha_map)

+    {

+	/* If the image has an alpha map of its own,

+	 * then it can't be used as an alpha map itself

+	 */

+	return;

+    }

+

     if (common->alpha_map != (bits_image_t *)alpha_map)

     {

 	if (common->alpha_map)

+	{

+	    common->alpha_map->common.alpha_count--;

+

 	    pixman_image_unref ((pixman_image_t *)common->alpha_map);

+	}

 

 	if (alpha_map)

+	{

 	    common->alpha_map = (bits_image_t *)pixman_image_ref (alpha_map);

+

+	    common->alpha_map->common.alpha_count++;

+	}

 	else

+	{

 	    common->alpha_map = NULL;

+	}

     }

 

     common->alpha_origin_x = x;

diff --git a/pixman/pixman/pixman-private.h b/pixman/pixman/pixman-private.h
index 6774b1514..0a6cbd275 100644
--- a/pixman/pixman/pixman-private.h
+++ b/pixman/pixman/pixman-private.h
@@ -80,6 +80,7 @@ struct image_common
     image_type_t                type;

     int32_t                     ref_count;

     pixman_region32_t           clip_region;

+    int32_t			alpha_count;	    /* How many times this image is being used as an alpha map */

     pixman_bool_t               have_clip_region;   /* FALSE if there is no clip */

     pixman_bool_t               client_clip;        /* Whether the source clip was

 						       set by a client */

@@ -570,6 +571,7 @@ _pixman_choose_implementation (void);
 #define FAST_PATH_BILINEAR_FILTER		(1 << 20)

 #define FAST_PATH_NO_NORMAL_REPEAT		(1 << 21)

 #define FAST_PATH_HAS_TRANSFORM			(1 << 22)

+#define FAST_PATH_SAMPLES_OPAQUE		(1 << 23)

 

 #define FAST_PATH_PAD_REPEAT						\

     (FAST_PATH_NO_NONE_REPEAT		|				\

diff --git a/pixman/pixman/pixman.c b/pixman/pixman/pixman.c
index 1dbfd83f8..819bb5146 100644
--- a/pixman/pixman/pixman.c
+++ b/pixman/pixman/pixman.c
@@ -884,6 +884,17 @@ do_composite (pixman_op_t	       op,
     if (!analyze_extent (mask, dest_x - mask_x, dest_y - mask_y, extents, &mask_flags))

 	goto out;

 

+    /* If the clip is within the source samples, and the samples are opaque,

+     * then the source is effectively opaque.

+     */

+#define BOTH (FAST_PATH_SAMPLES_OPAQUE | FAST_PATH_SAMPLES_COVER_CLIP)

+

+    if ((src_flags & BOTH) == BOTH)

+	src_flags |= FAST_PATH_IS_OPAQUE;

+    

+    if ((mask_flags & BOTH) == BOTH)

+	mask_flags |= FAST_PATH_IS_OPAQUE;

+    

     /*

      * Check if we can replace our operator by a simpler one

      * if the src or dest are opaque. The output operator should be

diff --git a/pixman/test/Makefile.am b/pixman/test/Makefile.am
index efd9cd91a..d3e9d3fa0 100644
--- a/pixman/test/Makefile.am
+++ b/pixman/test/Makefile.am
@@ -13,6 +13,7 @@ TESTPROGRAMS =			\
 	gradient-crash-test	\

 	trap-crasher		\

 	alphamap		\

+	alpha-loop		\

 	scaling-crash-test	\

 	blitters-test		\

 	scaling-test		\

@@ -39,6 +40,9 @@ scaling_test_SOURCES = scaling-test.c utils.c utils.h
 alphamap_LDADD = $(TEST_LDADD)

 alphamap_SOURCES = alphamap.c utils.c utils.h

 

+alpha_loop_LDADD = $(TEST_LDADD)

+alpha_loop_SOURCES = alpha-loop.c utils.c utils.h

+

 # GTK using test programs

 

 if HAVE_GTK

diff --git a/pixman/test/alpha-loop.c b/pixman/test/alpha-loop.c
new file mode 100644
index 000000000..38fc21f5d
--- /dev/null
+++ b/pixman/test/alpha-loop.c
@@ -0,0 +1,29 @@
+#include <stdio.h>

+#include <stdlib.h>

+#include "utils.h"

+

+#define WIDTH 400

+#define HEIGHT 200

+

+int

+main (int argc, char **argv)

+{

+    uint8_t *alpha = make_random_bytes (WIDTH * HEIGHT);

+    uint32_t *src = (uint32_t *)make_random_bytes (WIDTH * HEIGHT * 4);

+    uint32_t *dest = (uint32_t *)make_random_bytes (WIDTH * HEIGHT * 4);

+

+    pixman_image_t *a = pixman_image_create_bits (PIXMAN_a8, WIDTH, HEIGHT, (uint32_t *)alpha, WIDTH);

+    pixman_image_t *d = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, dest, WIDTH * 4);

+    pixman_image_t *s = pixman_image_create_bits (PIXMAN_a2r10g10b10, WIDTH, HEIGHT, src, WIDTH * 4);

+

+    fail_after (5, "Infinite loop detected: 5 seconds without progress\n");

+

+    pixman_image_set_alpha_map (s, a, 0, 0);

+    pixman_image_set_alpha_map (a, s, 0, 0);

+

+    pixman_image_composite (PIXMAN_OP_SRC, s, NULL, d, 0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);

+

+    pixman_image_unref (s);

+

+    return 0;

+}

diff --git a/pixman/test/utils.c b/pixman/test/utils.c
index 4b68debff..f5199268b 100644
--- a/pixman/test/utils.c
+++ b/pixman/test/utils.c
@@ -1,4 +1,9 @@
 #include "utils.h"

+#include <signal.h>

+

+#ifdef HAVE_UNISTD_H

+#include <unistd.h>

+#endif

 

 /* Random number seed

  */

@@ -319,3 +324,31 @@ fuzzer_test_main (const char *test_name,
 

     return 0;

 }

+

+static const char *global_msg;

+

+static void

+on_alarm (int signo)

+{

+    printf ("%s\n", global_msg);

+    exit (1);

+}

+

+void

+fail_after (int seconds, const char *msg)

+{

+#ifdef HAVE_SIGACTION

+#ifdef HAVE_ALARM

+    struct sigaction action;

+

+    global_msg = msg;

+

+    memset (&action, 0, sizeof (action));

+    action.sa_handler = on_alarm;

+

+    alarm (seconds);

+

+    sigaction (SIGALRM, &action, NULL);

+#endif

+#endif

+}

diff --git a/pixman/test/utils.h b/pixman/test/utils.h
index 39b49b2c1..a5b3c91c4 100644
--- a/pixman/test/utils.h
+++ b/pixman/test/utils.h
@@ -62,3 +62,6 @@ fuzzer_test_main (const char *test_name,
 		  uint32_t    (*test_function)(int testnum, int verbose),

 		  int         argc,

 		  const char *argv[]);

+

+void

+fail_after (int seconds, const char *msg);