// Dual vertical pass scaler for ARM NEON.
// Written by Nils Liaeen Corneliusen.
// License: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication license
// https://www.ignorantus.com/

// YUV420 planar:
// 0480*0270: 4770 (209.64fps)
// 0960*0540: 8601 (116.27fps)
// 1280*0720: 12277 (81.45fps)
// 1440*0810: 14496 (68.98fps)
// 1920*1080: 21504 (46.50fps)

// NV12 (Y planar, UV interleaved):
// 0480*0270: 4398 (227.38fps)
// 0960*0540: 7364 (135.80fps)
// 1280*0720: 10589 (94.44fps)
// 1440*0810: 12556 (79.64fps)
// 1920*1080: 18388 (54.38fps)


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "scale_neon.h"
#include "coeffs.h"

#define SCALE_SHIFT 6
#define ROUNDVAL    (1<<(SCALE_SHIFT-1))


#define GETOFFSET(p) ( ((p)>>3)*64+((p)&0x07) )

static void vert_interpolate_4to1_8( uint8_t *src, int src_stride, uint32_t srch, int yy0,
                                     uint8_t *dst, int width, uint32_t *ycoeffs )
{
    uint8_t *sp0 = src + src_stride * _min_s32(_max_s32((yy0>>16)-1,0), srch);
    uint8_t *sp1 = src + src_stride * _min_s32(_max_s32((yy0>>16)+0,0), srch);
    uint8_t *sp2 = src + src_stride * _min_s32(_max_s32((yy0>>16)+1,0), srch);
    uint8_t *sp3 = src + src_stride * _min_s32(_max_s32((yy0>>16)+2,0), srch);

    uint8_t *dst0;
    uint32_t ws;

    // q0-3 vsums, s16
    // q4 rounding constant
    // q5 1 coeff expanded, s16
    // q6 data expanded
    // q7 coeffs, s16
    // q8-q15 src
    asm volatile (

        "pld       [%[sp0], #0]\n\t"
        "pld       [%[sp1], #0]\n\t"
        "pld       [%[sp2], #0]\n\t"
        "pld       [%[sp3], #0]\n\t"
        "pld       [%[sp0], #64]\n\t"
        "pld       [%[sp1], #64]\n\t"
        "pld       [%[sp2], #64]\n\t"
        "pld       [%[sp3], #64]\n\t"

        "sub      %[ws],   %[width], #1\n\t"
        "mov      %[dst0], %[dst]\n\t"

        // co0 = *(coeffs + ((((uint32_t)yy0)>>10)&63));
        "ubfx     %[yy0], %[yy0], #10, #6\n\t"
        "add      %[yy0], %[ycoeffs], %[yy0], lsl #2\n\t"
        "vld1.32  d14, [%[yy0]]\n\t"
        "vmovl.s8 q7, d14\n\t"

        "vmov.u16 q4, #0x0020\n\t"

        "vld1.64   {d16-d19}, [%[sp0]:64]!\n\t"
        "vld1.64   {d20-d23}, [%[sp1]:64]!\n\t"

        "32421:\n\t"

        "pld       [%[sp0], #128]\n\t"
        "pld       [%[sp1], #128]\n\t"
        "pld       [%[sp2], #128]\n\t"
        "pld       [%[sp3], #128]\n\t"

        "vdup.s16  q5,  d14[0]\n\t"
        "vmovl.u8  q12, d16\n\t"
        "vmovl.u8  q13, d17\n\t"
        "vmovl.u8  q14, d18\n\t"
        "vmovl.u8  q15, d19\n\t"
        "vmul.s16  q0,  q5, q12\n\t"
        "vmul.s16  q1,  q5, q13\n\t"
        "vmul.s16  q2,  q5, q14\n\t"
        "vmul.s16  q3,  q5, q15\n\t"

        "vdup.s16  q5,  d14[1]\n\t"
        "vmovl.u8  q12, d20\n\t"
        "vmovl.u8  q13, d21\n\t"
        "vmovl.u8  q14, d22\n\t"
        "vmovl.u8  q15, d23\n\t"
        "vmla.s16  q0,  q5, q12\n\t"
        "vmla.s16  q1,  q5, q13\n\t"
        "vmla.s16  q2,  q5, q14\n\t"
        "vmla.s16  q3,  q5, q15\n\t"

        "vld1.64   {d24-d27}, [%[sp2]:64]!\n\t"
        "vld1.64   {d28-d31}, [%[sp3]:64]!\n\t"

        "vdup.s16  q5,  d14[2]\n\t"
        "vmovl.u8  q8,  d24\n\t"
        "vmovl.u8  q9,  d25\n\t"
        "vmovl.u8  q10, d26\n\t"
        "vmovl.u8  q11, d27\n\t"
        "vmla.s16  q0,  q5, q8\n\t"
        "vmla.s16  q1,  q5, q9\n\t"
        "vmla.s16  q2,  q5, q10\n\t"
        "vmla.s16  q3,  q5, q11\n\t"

        "vdup.s16  q5,  d14[3]\n\t"
        "vmovl.u8  q8,  d28\n\t"
        "vmovl.u8  q9,  d29\n\t"
        "vmovl.u8  q10, d30\n\t"
        "vmovl.u8  q11, d31\n\t"
        "vmla.s16  q0,  q5, q8\n\t"
        "vmla.s16  q1,  q5, q9\n\t"
        "vmla.s16  q2,  q5, q10\n\t"
        "vmla.s16  q3,  q5, q11\n\t"

        "vld1.64   {d16-d19}, [%[sp0]:64]!\n\t"
        "vld1.64   {d20-d23}, [%[sp1]:64]!\n\t"

        // round and pack
        "vadd.s16 q0, q0, q4\n\t"
        "vadd.s16 q1, q1, q4\n\t"
        "vadd.s16 q2, q2, q4\n\t"
        "vadd.s16 q3, q3, q4\n\t" //n.71-0   1c n0 q3:3

        "vqshrun.s16 d0, q0, #6\n\t"
        "vqshrun.s16 d1, q1, #6\n\t"
        "vqshrun.s16 d2, q2, #6\n\t"
        "vqshrun.s16 d3, q3, #6\n\t"

        "vstr     d0, [%[dst], #0]\n\t"
        "vstr     d1, [%[dst], #64]\n\t"
        "vstr     d2, [%[dst], #128]\n\t"
        "vstr     d3, [%[dst], #192]\n\t"

        "subs     %[width], #32\n\t"
        "add      %[dst], #256\n\t"
        "bgt      32421b\n\t"

        // left pad
        // *(uint64_t *)(dst0-64) = dup8( dst0[0] );
        "ldrb    %[sp0], [%[dst0]]\n\t"
        "vdup.8  d0, %[sp0]\n\t"
        "vstr    d0, [%[dst0], #-64]\n\t"

        // right pad
        // avoid unaligned stores when srcw is odd
        // pad = *(dst0+64+GETOFFSET(ws));
        "mov     %[sp0], %[ws], lsr #3\n\t"
        "lsl     %[sp0], %[sp0], #6\n\t"
        "and     %[sp2], %[ws], #7\n\t"
        "add     %[sp0], %[sp0], %[sp2]\n\t"
        "ldrb    %[sp1], [%[dst0], %[sp0]]\n\t"

        // *(dst0+64+GETOFFSET(ws+1)) = pad;
        "add     %[ws],  %[ws], #1\n\t"
        "lsr     %[sp0], %[ws], #3\n\t"
        "lsl     %[sp0], %[sp0], #6\n\t"
        "and     %[sp2], %[ws], #7\n\t"
        "add     %[sp0], %[sp0], %[sp2]\n\t"
        "strb    %[sp1], [%[dst0], %[sp0]]\n\t"

        // *(dst0+64+GETOFFSET(ws+2)) = pad;
        "add     %[ws],  %[ws], #1\n\t"
        "lsr     %[sp0], %[ws], #3\n\t"
        "lsl     %[sp0], %[sp0], #6\n\t"
        "and     %[sp2], %[ws], #7\n\t"
        "add     %[sp0], %[sp0], %[sp2]\n\t"
        "strb    %[sp1], [%[dst0], %[sp0]]\n\t"

    : [dst] "+r" (dst), [dst0] "+r" (dst0),
      [sp0] "+r" (sp0), [sp1] "+r" (sp1), [sp2] "+r" (sp2), [sp3] "+r" (sp3),
      [ws] "+r" (ws), [yy0] "+r" (yy0), [width] "+r" (width)
    : [ycoeffs] "r" (ycoeffs)
    :  "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
       "d8", "d9","d10","d11","d12","d13","d14","d15",
      "d16","d17","d18","d19","d20","d21","d22","d23",
      "d24","d25","d26","d27","d28","d29","d30","d31"
    );

}


// Just need to shuffle enough to get pixels on correct row.
// col 0: 00,10,20,30,40,50,60,70 => row 0: 00,40,10,50,20,60,30,70
static inline void rot90left( uint8_t *src, int cnt )
{
    uint8_t *dst;

    asm volatile (
        "mov     %[dst], %[src]\n\t"

        "vld4.8  {d0-d3},   [%[src]:64]!\n\t"
        "vld4.8  {d4-d7},   [%[src]:64]!\n\t"

        "9016:\n\t"
        "subs    %[cnt], #16\n\t"

        "vtrn.8  q0, q2\n\t"
        "vtrn.8  q1, q3\n\t"

        "pld     [%[src], #192]\n\t"
        "pld     [%[src], #256]\n\t"

        "vld4.8  {d8-d11},  [%[src]:64]!\n\t"
        "vld4.8  {d12-d15}, [%[src]:64]!\n\t"

        "vst1.64 {d0-d3},   [%[dst]:64]!\n\t"
        "vst1.64 {d4-d7},   [%[dst]:64]!\n\t"

        "vtrn.8  q4, q6\n\t"
        "vtrn.8  q5, q7\n\t"

        "vld4.8  {d0-d3},   [%[src]:64]!\n\t"
        "vld4.8  {d4-d7},   [%[src]:64]!\n\t"

        "vst1.64 {d8-d11},   [%[dst]:64]!\n\t"
        "vst1.64 {d12-d15},  [%[dst]:64]!\n\t"

        "bgt     9016b\n\t"
    : [src] "+r" (src), [dst] "+r" (dst), [cnt] "+r" (cnt)
    :
    :  "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
       "d8", "d9","d10","d11","d12","d13","d14","d15"
    );

}


static void horiz_interpolate_8_8( uint8_t *src,
                                   int xx_fact, uint32_t *xcoeffs, int width,
                                   uint8_t *dst, int dst_stride )
{
    int xx_0;
    uint32_t tmp;

    // q0 coeffs, s16
    // q1 coeff duplicated, s16
    // q2 data expanded, u16
    // q4-q5 src data
    // q8-q15 wide results, s16
    asm volatile (

        // xx_0 = (xx_fact>>1)-(1<<15) - 65536;
        "mov     %[xx_0], %[xx_fact], asr #1\n\t"
        "sub     %[xx_0], %[xx_0], #98304\n\t"

        "88:\n\t"

        // 0
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q8, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #3\n\t"
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1, d0[0]\n\t"
        "vmovl.u8 q2, d8\n\t"
        "vmla.s16 q8, q1, q2\n\t"
        "vdup.s16 q1, d0[1]\n\t"
        "vmovl.u8 q2, d9\n\t"
        "vmla.s16 q8, q1, q2\n\t"
        "vdup.s16 q1, d0[2]\n\t"
        "vmovl.u8 q2, d10\n\t"
        "vmla.s16 q8, q1, q2\n\t"
        "vdup.s16 q1, d0[3]\n\t"
        "vmovl.u8 q2, d11\n\t"
        "vmla.s16 q8, q1, q2\n\t"

        // 1
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q9, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #3\n\t"
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1, d0[0]\n\t"
        "vmovl.u8 q2, d8\n\t"
        "vmla.s16 q9, q1, q2\n\t"
        "vdup.s16 q1, d0[1]\n\t"
        "vmovl.u8 q2, d9\n\t"
        "vmla.s16 q9, q1, q2\n\t"
        "vdup.s16 q1, d0[2]\n\t"
        "vmovl.u8 q2, d10\n\t"
        "vmla.s16 q9, q1, q2\n\t"
        "vdup.s16 q1, d0[3]\n\t"
        "vmovl.u8 q2, d11\n\t"
        "vmla.s16 q9, q1, q2\n\t"

        // 2
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q10, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #3\n\t"
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1, d0[0]\n\t"
        "vmovl.u8 q2, d8\n\t"
        "vmla.s16 q10, q1, q2\n\t"
        "vdup.s16 q1, d0[1]\n\t"
        "vmovl.u8 q2, d9\n\t"
        "vmla.s16 q10, q1, q2\n\t"
        "vdup.s16 q1, d0[2]\n\t"
        "vmovl.u8 q2, d10\n\t"
        "vmla.s16 q10, q1, q2\n\t"
        "vdup.s16 q1, d0[3]\n\t"
        "vmovl.u8 q2, d11\n\t"
        "vmla.s16 q10, q1, q2\n\t"

        // 3
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q11, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #3\n\t"
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1, d0[0]\n\t"
        "vmovl.u8 q2, d8\n\t"
        "vmla.s16 q11, q1, q2\n\t"
        "vdup.s16 q1, d0[1]\n\t"
        "vmovl.u8 q2, d9\n\t"
        "vmla.s16 q11, q1, q2\n\t"
        "vdup.s16 q1, d0[2]\n\t"
        "vmovl.u8 q2, d10\n\t"
        "vmla.s16 q11, q1, q2\n\t"
        "vdup.s16 q1, d0[3]\n\t"
        "vmovl.u8 q2, d11\n\t"
        "vmla.s16 q11, q1, q2\n\t"

        // 4
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q12, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #3\n\t"
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1, d0[0]\n\t"
        "vmovl.u8 q2, d8\n\t"
        "vmla.s16 q12, q1, q2\n\t"
        "vdup.s16 q1, d0[1]\n\t"
        "vmovl.u8 q2, d9\n\t"
        "vmla.s16 q12, q1, q2\n\t"
        "vdup.s16 q1, d0[2]\n\t"
        "vmovl.u8 q2, d10\n\t"
        "vmla.s16 q12, q1, q2\n\t"
        "vdup.s16 q1, d0[3]\n\t"
        "vmovl.u8 q2, d11\n\t"
        "vmla.s16 q12, q1, q2\n\t"

        // 5
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q13, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #3\n\t"
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1, d0[0]\n\t"
        "vmovl.u8 q2, d8\n\t"
        "vmla.s16 q13, q1, q2\n\t"
        "vdup.s16 q1, d0[1]\n\t"
        "vmovl.u8 q2, d9\n\t"
        "vmla.s16 q13, q1, q2\n\t"
        "vdup.s16 q1, d0[2]\n\t"
        "vmovl.u8 q2, d10\n\t"
        "vmla.s16 q13, q1, q2\n\t"
        "vdup.s16 q1, d0[3]\n\t"
        "vmovl.u8 q2, d11\n\t"
        "vmla.s16 q13, q1, q2\n\t"

        // 6
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q14, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #3\n\t"
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1, d0[0]\n\t"
        "vmovl.u8 q2, d8\n\t"
        "vmla.s16 q14, q1, q2\n\t"
        "vdup.s16 q1, d0[1]\n\t"
        "vmovl.u8 q2, d9\n\t"
        "vmla.s16 q14, q1, q2\n\t"
        "vdup.s16 q1, d0[2]\n\t"
        "vmovl.u8 q2, d10\n\t"
        "vmla.s16 q14, q1, q2\n\t"
        "vdup.s16 q1, d0[3]\n\t"
        "vmovl.u8 q2, d11\n\t"
        "vmla.s16 q14, q1, q2\n\t"

        // 7
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q15, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #3\n\t"
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1, d0[0]\n\t"
        "vmovl.u8 q2, d8\n\t"
        "vmla.s16 q15, q1, q2\n\t"
        "vdup.s16 q1, d0[1]\n\t"
        "vmovl.u8 q2, d9\n\t"
        "vmla.s16 q15, q1, q2\n\t"
        "vdup.s16 q1, d0[2]\n\t"
        "vmovl.u8 q2, d10\n\t"
        "vmla.s16 q15, q1, q2\n\t"
        "vdup.s16 q1, d0[3]\n\t"
        "vmovl.u8 q2, d11\n\t"
        "vmla.s16 q15, q1, q2\n\t"

        // very speculative...
        "pld     [%[tmp], #96]\n\t"
        "pld     [%[tmp], #160]\n\t"

        "vqshrun.s16 d16, q8,  #6\n\t"
        "vqshrun.s16 d17, q9,  #6\n\t"
        "vqshrun.s16 d18, q10, #6\n\t"
        "vqshrun.s16 d19, q11, #6\n\t"
        "vqshrun.s16 d20, q12, #6\n\t"
        "vqshrun.s16 d21, q13, #6\n\t"
        "vqshrun.s16 d22, q14, #6\n\t"
        "vqshrun.s16 d23, q15, #6\n\t"

        // Undo rot90left
        // or 6 vzip, but higher latency so it's slower
        "vtrn.8   d16, d17\n\t"
        "vtrn.8   d18, d19\n\t"
        "vtrn.8   d20, d21\n\t"
        "vtrn.8   d22, d23\n\t" // n.183-0  1c n0 d23:3
        "vtrn.16  q8,  q9\n\t"
        "vtrn.16  q10, q11\n\t"
        "vtrn.32  q8,  q10\n\t"
        "vtrn.32  q9,  q11\n\t"

        "mov      %[tmp], %[dst]\n\t"

        "vst1.64  d16, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d18, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d20, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d22, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d17, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d19, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d21, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d23, [%[tmp]:64], %[dst_stride]\n\t"

        "subs     %[width], %[width], #8\n\t"
        "add      %[dst], %[dst], #8\n\t"
        "bgt      88b\n\t"

    : [tmp] "+r" (tmp), [xx_0] "+r" (xx_0), [dst] "+r" (dst), [width] "+r" (width)
    : [src] "r" (src), [xcoeffs] "r" (xcoeffs), [xx_fact] "r" (xx_fact), [dst_stride] "r" (dst_stride)
    :   "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
        "d8", "d9","d10","d11","d12","d13","d14","d15",
       "d16","d17","d18","d19","d20","d21","d22","d23",
       "d24","d25","d26","d27","d28","d29","d30","d31"
    );

}


void bicubic_scale_8( scalefunc_data_t *d )
{
    int srcw       = d->srcw;
    int dstw       = d->dstw;
    int src_stride = d->src_stride;
    int dst_stride = d->dst_stride;
    int yy_fact    = d->yy_fact;
    int xx_fact    = d->xx_fact;

    uint32_t *xcoeffs = scale_get_filter_coeffs32( xx_fact );
    uint32_t *ycoeffs = scale_get_filter_coeffs32( yy_fact );

    // +16 is left/right pad, +32 is write over right edge in vert
    uint8_t *hbuf8 = (uint8_t *)alloca( ((srcw+16+32+63)&(~63))*8 );
    hbuf8 += 64;

    int yy0 = (yy_fact>>1)-(1<<15) + d->y_start*yy_fact;

    for( int y = d->y_start; y < d->y_start + d->height; y += 8 ) {

        for( int i = 0; i < 8; i++ ) {

            vert_interpolate_4to1_8( d->src, src_stride, d->srch-1, yy0, hbuf8+i*8, srcw, ycoeffs );

            yy0 += yy_fact;
        }

        rot90left( hbuf8-64, srcw+16 );

        horiz_interpolate_8_8( hbuf8, xx_fact, xcoeffs, dstw, d->dst + y*dst_stride, dst_stride );

    }
}


void bicubic_scale_yuv420( bmp_yuv420pl *src, bmp_yuv420pl *dst )
{
    scalefunc_data_t sd;

    sd.y_start = 0;
    sd.height  = dst->h;

    sd.xx_fact = (src->w<<16)/dst->w;
    sd.yy_fact = (src->h<<16)/dst->h;

    sd.src_stride = src->ystride;
    sd.srcw = src->w;
    sd.srch = src->h;
    sd.dst_stride = dst->ystride;
    sd.dstw = dst->w;
    sd.dsth = dst->h;

    sd.src = src->y;
    sd.dst = dst->y;
    bicubic_scale_8( &sd );

    sd.srcw = src->w/2;
    sd.srch = src->h/2;
    sd.dstw = dst->w/2;
    sd.dsth = dst->h/2;
    sd.src_stride = src->uvstride;
    sd.dst_stride = dst->uvstride;

    sd.height  = dst->h/2;

    sd.src = src->u;
    sd.dst = dst->u;
    bicubic_scale_8( &sd );

    sd.src = src->v;
    sd.dst = dst->v;
    bicubic_scale_8( &sd );
}


static void vert_interpolate_4to1_16( uint8_t *src, int src_stride, uint32_t srch, int yy0,
                                      uint8_t *dst, int width, uint32_t *ycoeffs )
{
    uint8_t *sp0 = src + src_stride * _min_s32(_max_s32((yy0>>16)-1,0), srch);
    uint8_t *sp1 = src + src_stride * _min_s32(_max_s32((yy0>>16)+0,0), srch);
    uint8_t *sp2 = src + src_stride * _min_s32(_max_s32((yy0>>16)+1,0), srch);
    uint8_t *sp3 = src + src_stride * _min_s32(_max_s32((yy0>>16)+2,0), srch);

    uint8_t *dst0;
    uint32_t ws;

    // q0-3 vsums, s16
    // q4 rounding constant
    // q5 1 coeff expanded, s16
    // q6 data expanded
    // q7 coeffs, s16
    // q8-q15 src
    asm volatile (

        "pld       [%[sp0], #0]\n\t"
        "pld       [%[sp1], #0]\n\t"
        "pld       [%[sp2], #0]\n\t"
        "pld       [%[sp3], #0]\n\t"
        "pld       [%[sp0], #64]\n\t"
        "pld       [%[sp1], #64]\n\t"
        "pld       [%[sp2], #64]\n\t"
        "pld       [%[sp3], #64]\n\t"

        "sub      %[ws],   %[width], #2\n\t"
        "mov      %[dst0], %[dst]\n\t"

        // co0 = *(coeffs + ((((uint32_t)yy0)>>10)&63));
        "ubfx     %[yy0], %[yy0], #10, #6\n\t"
        "add      %[yy0], %[ycoeffs], %[yy0], lsl #2\n\t"
        "vld1.32  d14, [%[yy0]]\n\t"
        "vmovl.s8 q7, d14\n\t"

        "vmov.u16 q4, #0x0020\n\t"

        "vld1.64   {d16-d19}, [%[sp0]:64]!\n\t"
        "vld1.64   {d20-d23}, [%[sp1]:64]!\n\t"

        "3242112:\n\t"

        "pld       [%[sp0], #128]\n\t"
        "pld       [%[sp1], #128]\n\t"
        "pld       [%[sp2], #128]\n\t"
        "pld       [%[sp3], #128]\n\t"

        "vdup.s16  q5,  d14[0]\n\t"
        "vmovl.u8  q12, d16\n\t"
        "vmovl.u8  q13, d17\n\t"
        "vmovl.u8  q14, d18\n\t"
        "vmovl.u8  q15, d19\n\t"
        "vmul.s16  q0,  q5, q12\n\t"
        "vmul.s16  q1,  q5, q13\n\t"
        "vmul.s16  q2,  q5, q14\n\t"
        "vmul.s16  q3,  q5, q15\n\t"

        "vdup.s16  q5,  d14[1]\n\t"
        "vmovl.u8  q12, d20\n\t"
        "vmovl.u8  q13, d21\n\t"
        "vmovl.u8  q14, d22\n\t"
        "vmovl.u8  q15, d23\n\t"
        "vmla.s16  q0,  q5, q12\n\t"
        "vmla.s16  q1,  q5, q13\n\t"
        "vmla.s16  q2,  q5, q14\n\t"
        "vmla.s16  q3,  q5, q15\n\t"

        "vld1.64   {d24-d27}, [%[sp2]:64]!\n\t"
        "vld1.64   {d28-d31}, [%[sp3]:64]!\n\t"

        "vdup.s16  q5,  d14[2]\n\t"
        "vmovl.u8  q8,  d24\n\t"
        "vmovl.u8  q9,  d25\n\t"
        "vmovl.u8  q10, d26\n\t"
        "vmovl.u8  q11, d27\n\t"
        "vmla.s16  q0,  q5, q8\n\t"
        "vmla.s16  q1,  q5, q9\n\t"
        "vmla.s16  q2,  q5, q10\n\t"
        "vmla.s16  q3,  q5, q11\n\t"

        "vdup.s16  q5,  d14[3]\n\t"
        "vmovl.u8  q8,  d28\n\t"
        "vmovl.u8  q9,  d29\n\t"
        "vmovl.u8  q10, d30\n\t"
        "vmovl.u8  q11, d31\n\t"
        "vmla.s16  q0,  q5, q8\n\t"
        "vmla.s16  q1,  q5, q9\n\t"
        "vmla.s16  q2,  q5, q10\n\t"
        "vmla.s16  q3,  q5, q11\n\t"

        "vld1.64   {d16-d19}, [%[sp0]:64]!\n\t"
        "vld1.64   {d20-d23}, [%[sp1]:64]!\n\t"

        // round and pack
        "vadd.s16 q0, q0, q4\n\t"
        "vadd.s16 q1, q1, q4\n\t"
        "vadd.s16 q2, q2, q4\n\t"
        "vadd.s16 q3, q3, q4\n\t" //n.71-0   1c n0 q3:3

        "vqshrun.s16 d0, q0, #6\n\t"
        "vqshrun.s16 d1, q1, #6\n\t"
        "vqshrun.s16 d2, q2, #6\n\t"
        "vqshrun.s16 d3, q3, #6\n\t"

        "vstr     d0, [%[dst], #0]\n\t"
        "vstr     d1, [%[dst], #64]\n\t"
        "vstr     d2, [%[dst], #128]\n\t"
        "vstr     d3, [%[dst], #192]\n\t"

        "subs     %[width], #32\n\t"
        "add      %[dst], #256\n\t"
        "bgt      3242112b\n\t"

        // left pad
        // *(uint64_t *)(dst0-64) = dup16( dst0[0] );
        "ldrh    %[sp0], [%[dst0]]\n\t"
        "vdup.16 d0, %[sp0]\n\t"
        "vstr    d0, [%[dst0], #-64]\n\t"

        // right pad
        // avoid unaligned stores when srcw is odd
        // pad = *(dst0+64+GETOFFSET(ws));
        "mov     %[sp0], %[ws], lsr #3\n\t"
        "lsl     %[sp0], %[sp0], #6\n\t"
        "and     %[sp2], %[ws], #7\n\t"
        "add     %[sp0], %[sp0], %[sp2]\n\t"
        "ldrh    %[sp1], [%[dst0], %[sp0]]\n\t"

        // *(dst0+64+GETOFFSET(ws+2)) = pad;
        "add     %[ws],  %[ws], #2\n\t"
        "lsr     %[sp0], %[ws], #3\n\t"
        "lsl     %[sp0], %[sp0], #6\n\t"
        "and     %[sp2], %[ws], #7\n\t"
        "add     %[sp0], %[sp0], %[sp2]\n\t"
        "strh    %[sp1], [%[dst0], %[sp0]]\n\t"

        // *(dst0+64+GETOFFSET(ws+4)) = pad;
        "add     %[ws],  %[ws], #2\n\t"
        "lsr     %[sp0], %[ws], #3\n\t"
        "lsl     %[sp0], %[sp0], #6\n\t"
        "and     %[sp2], %[ws], #7\n\t"
        "add     %[sp0], %[sp0], %[sp2]\n\t"
        "strh    %[sp1], [%[dst0], %[sp0]]\n\t"

    : [dst] "+r" (dst), [dst0] "+r" (dst0),
      [sp0] "+r" (sp0), [sp1] "+r" (sp1), [sp2] "+r" (sp2), [sp3] "+r" (sp3),
      [ws] "+r" (ws), [yy0] "+r" (yy0), [width] "+r" (width)
    : [ycoeffs] "r" (ycoeffs)
    :  "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
       "d8", "d9","d10","d11","d12","d13","d14","d15",
      "d16","d17","d18","d19","d20","d21","d22","d23",
      "d24","d25","d26","d27","d28","d29","d30","d31"
    );

}


static void horiz_interpolate_8_16( uint8_t *src,
                                    int xx_fact, uint32_t *xcoeffs, int width,
                                    uint8_t *dst, int dst_stride )
{
    int xx_0;
    uint32_t tmp;

    // q0 coeffs, s16
    // q1 coeff duplicated, s16
    // q2-q3 data expanded, u16
    // q4-q7 src data
    // q8-q15 wide results, s16
    asm volatile (

        // xx_0 = (xx_fact>>1)-(1<<15) - 65536;
        "mov     %[xx_0], %[xx_fact], asr #1\n\t"
        "sub     %[xx_0], %[xx_0], #98304\n\t"

        "84:\n\t"

        // 0
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q8, #0x0020\n\t"
        "vmov.i16 q9, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #4\n\t" //
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11},  [%[tmp]:64]!\n\t"
        "vld1.64  {d12-d15}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1, d0[0]\n\t"
        "vmovl.u8 q2, d8\n\t"
        "vmovl.u8 q3, d9\n\t"
        "vmla.s16 q8, q1, q2\n\t"
        "vmla.s16 q9, q1, q3\n\t"
        "vdup.s16 q1, d0[1]\n\t"
        "vmovl.u8 q2, d10\n\t"
        "vmovl.u8 q3, d11\n\t"
        "vmla.s16 q8, q1, q2\n\t"
        "vmla.s16 q9, q1, q3\n\t"
        "vdup.s16 q1, d0[2]\n\t"
        "vmovl.u8 q2, d12\n\t"
        "vmovl.u8 q3, d13\n\t"
        "vmla.s16 q8, q1, q2\n\t"
        "vmla.s16 q9, q1, q3\n\t"
        "vdup.s16 q1, d0[3]\n\t"
        "vmovl.u8 q2, d14\n\t"
        "vmovl.u8 q3, d15\n\t"
        "vmla.s16 q8, q1, q2\n\t"
        "vmla.s16 q9, q1, q3\n\t"

        // 1
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q10, #0x0020\n\t"
        "vmov.i16 q11, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #4\n\t" //
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11},  [%[tmp]:64]!\n\t"
        "vld1.64  {d12-d15}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1,  d0[0]\n\t"
        "vmovl.u8 q2,  d8\n\t"
        "vmovl.u8 q3,  d9\n\t"
        "vmla.s16 q10, q1, q2\n\t"
        "vmla.s16 q11, q1, q3\n\t"
        "vdup.s16 q1,  d0[1]\n\t"
        "vmovl.u8 q2,  d10\n\t"
        "vmovl.u8 q3,  d11\n\t"
        "vmla.s16 q10, q1, q2\n\t"
        "vmla.s16 q11, q1, q3\n\t"
        "vdup.s16 q1,  d0[2]\n\t"
        "vmovl.u8 q2,  d12\n\t"
        "vmovl.u8 q3,  d13\n\t"
        "vmla.s16 q10, q1, q2\n\t"
        "vmla.s16 q11, q1, q3\n\t"
        "vdup.s16 q1,  d0[3]\n\t"
        "vmovl.u8 q2,  d14\n\t"
        "vmovl.u8 q3,  d15\n\t"
        "vmla.s16 q10, q1, q2\n\t"
        "vmla.s16 q11, q1, q3\n\t"

        // 2
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q12, #0x0020\n\t"
        "vmov.i16 q13, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #4\n\t" //
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11},  [%[tmp]:64]!\n\t"
        "vld1.64  {d12-d15}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1,  d0[0]\n\t"
        "vmovl.u8 q2,  d8\n\t"
        "vmovl.u8 q3,  d9\n\t"
        "vmla.s16 q12, q1, q2\n\t"
        "vmla.s16 q13, q1, q3\n\t"
        "vdup.s16 q1,  d0[1]\n\t"
        "vmovl.u8 q2,  d10\n\t"
        "vmovl.u8 q3,  d11\n\t"
        "vmla.s16 q12, q1, q2\n\t"
        "vmla.s16 q13, q1, q3\n\t"
        "vdup.s16 q1,  d0[2]\n\t"
        "vmovl.u8 q2,  d12\n\t"
        "vmovl.u8 q3,  d13\n\t"
        "vmla.s16 q12, q1, q2\n\t"
        "vmla.s16 q13, q1, q3\n\t"
        "vdup.s16 q1,  d0[3]\n\t"
        "vmovl.u8 q2,  d14\n\t"
        "vmovl.u8 q3,  d15\n\t"
        "vmla.s16 q12, q1, q2\n\t"
        "vmla.s16 q13, q1, q3\n\t"

        // 3
        "ubfx     %[tmp], %[xx_0], #10, #6\n\t"
        "vmov.i16 q14, #0x0020\n\t"
        "vmov.i16 q15, #0x0020\n\t"
        "add      %[tmp], %[xcoeffs], %[tmp], lsl #2\n\t"
        "vld1.32  d0, [%[tmp]]\n\t"
        "asr      %[tmp], %[xx_0], #16\n\t"
        "add      %[tmp], %[src], %[tmp], lsl #4\n\t" //
        "vmovl.s8 q0, d0\n\t"
        "vld1.64  {d8-d11},  [%[tmp]:64]!\n\t"
        "vld1.64  {d12-d15}, [%[tmp]:64]\n\t"
        "add      %[xx_0], %[xx_0], %[xx_fact]\n\t"
        "vdup.s16 q1,  d0[0]\n\t"
        "vmovl.u8 q2,  d8\n\t"
        "vmovl.u8 q3,  d9\n\t"
        "vmla.s16 q14, q1, q2\n\t"
        "vmla.s16 q15, q1, q3\n\t"
        "vdup.s16 q1,  d0[1]\n\t"
        "vmovl.u8 q2,  d10\n\t"
        "vmovl.u8 q3,  d11\n\t"
        "vmla.s16 q14, q1, q2\n\t"
        "vmla.s16 q15, q1, q3\n\t"
        "vdup.s16 q1,  d0[2]\n\t"
        "vmovl.u8 q2,  d12\n\t"
        "vmovl.u8 q3,  d13\n\t"
        "vmla.s16 q14, q1, q2\n\t"
        "vmla.s16 q15, q1, q3\n\t"
        "vdup.s16 q1,  d0[3]\n\t"
        "vmovl.u8 q2,  d14\n\t"
        "vmovl.u8 q3,  d15\n\t"
        "vmla.s16 q14, q1, q2\n\t"
        "vmla.s16 q15, q1, q3\n\t"

        // very speculative...
        "pld     [%[tmp], #96]\n\t"
        "pld     [%[tmp], #160]\n\t"

        "vqshrun.s16 d16, q8,  #6\n\t"
        "vqshrun.s16 d17, q9,  #6\n\t"
        "vqshrun.s16 d18, q10, #6\n\t"
        "vqshrun.s16 d19, q11, #6\n\t"
        "vqshrun.s16 d20, q12, #6\n\t"
        "vqshrun.s16 d21, q13, #6\n\t"
        "vqshrun.s16 d22, q14, #6\n\t"
        "vqshrun.s16 d23, q15, #6\n\t"

        // Undo rot90left
        "vtrn.8   d16, d17\n\t"
        "vtrn.8   d18, d19\n\t"
        "vtrn.8   d20, d21\n\t"
        "vtrn.8   d22, d23\n\t"
        "vtrn.16  q8,  q9\n\t"
        "vtrn.16  q10, q11\n\t"
        "vtrn.32  q8,  q10\n\t"
        "vtrn.32  q9,  q11\n\t"

        "mov      %[tmp], %[dst]\n\t"

        "vst1.64  d16, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d18, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d20, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d22, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d17, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d19, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d21, [%[tmp]:64], %[dst_stride]\n\t"
        "vst1.64  d23, [%[tmp]:64], %[dst_stride]\n\t"

        "subs     %[width], %[width], #4\n\t"
        "add      %[dst], %[dst], #8\n\t"
        "bgt      84b\n\t"

    : [tmp] "+r" (tmp), [xx_0] "+r" (xx_0), [dst] "+r" (dst), [width] "+r" (width)
    : [src] "r" (src), [xcoeffs] "r" (xcoeffs), [xx_fact] "r" (xx_fact), [dst_stride] "r" (dst_stride)
    :   "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
        "d8", "d9","d10","d11","d12","d13","d14","d15",
       "d16","d17","d18","d19","d20","d21","d22","d23",
       "d24","d25","d26","d27","d28","d29","d30","d31"
    );

}


void bicubic_scale_16( scalefunc_data_t *d )
{
    int srcw       = d->srcw;
    int dstw       = d->dstw;
    int src_stride = d->src_stride;
    int dst_stride = d->dst_stride;
    int yy_fact    = d->yy_fact;
    int xx_fact    = d->xx_fact;

    uint32_t *xcoeffs = scale_get_filter_coeffs32( xx_fact );
    uint32_t *ycoeffs = scale_get_filter_coeffs32( yy_fact );

    // +16 is left/right pad, +32 is write over right edge in vert
    uint8_t *hbuf8 = (uint8_t *)alloca( ((srcw*2+16+32+63)&(~63))*8 );
    hbuf8 += 64;

    int yy0 = (yy_fact>>1)-(1<<15) + d->y_start*yy_fact;

    for( int y = d->y_start; y < d->y_start + d->height; y += 8 ) {

        for( int i = 0; i < 8; i++ ) {

            vert_interpolate_4to1_16( d->src, src_stride, d->srch-1, yy0, hbuf8+i*8, srcw*2, ycoeffs );

            yy0 += yy_fact;
        }

        rot90left( hbuf8-64, srcw*2+16 );

        horiz_interpolate_8_16( hbuf8, xx_fact, xcoeffs, dstw, d->dst + y*dst_stride, dst_stride );

    }
}


void bicubic_scale_nv12( bmp_nv12 *src, bmp_nv12 *dst )
{
    scalefunc_data_t sd;

    sd.y_start = 0;
    sd.height  = dst->h;

    sd.xx_fact = (src->w<<16)/dst->w;
    sd.yy_fact = (src->h<<16)/dst->h;

    // normal y scaling
    sd.src_stride = src->ystride;
    sd.srcw = src->w;
    sd.srch = src->h;
    sd.dst_stride = dst->ystride;
    sd.dstw = dst->w;
    sd.dsth = dst->h;

    sd.src = src->y;
    sd.dst = dst->y;
    bicubic_scale_8( &sd );

    // uv scaling
    sd.srcw = src->w/2;
    sd.srch = src->h/2;
    sd.dstw = dst->w/2;
    sd.dsth = dst->h/2;
    sd.src_stride = src->uvstride;
    sd.dst_stride = dst->uvstride;
    sd.height  = dst->h/2;
    sd.src = src->uv;
    sd.dst = dst->uv;
    bicubic_scale_16( &sd );
}


void bicubic_scale_rgb888pl( bmp_rgb888pl *src, bmp_rgb888pl *dst )
{
    scalefunc_data_t sd;

    sd.y_start = 0;
    sd.height  = dst->h;

    sd.xx_fact = (src->w<<16)/dst->w;
    sd.yy_fact = (src->h<<16)/dst->h;

    sd.src_stride = src->stride;
    sd.srcw = src->w;
    sd.srch = src->h;
    sd.dst_stride = dst->stride;
    sd.dstw = dst->w;
    sd.dsth = dst->h;

    sd.src = src->r;
    sd.dst = dst->r;
    bicubic_scale_8( &sd );

    sd.src = src->g;
    sd.dst = dst->g;
    bicubic_scale_8( &sd );

    sd.src = src->b;
    sd.dst = dst->b;
    bicubic_scale_8( &sd );

}


void bicubic_scale_yuv422seq_to_nv12( bmp_yuv422seq *src, bmp_nv12 *dst )
{
    // new vert_interpolate that splits and downsamples into y+uv
    // use regular horiz8/16 afterwards
}