- Main Page
- Related Pages
- Classes
- Files

- File List

# vectorize_neon.h

    00001 #ifndef OPENMM_VECTORIZE_NEON_H_
    00002 #define OPENMM_VECTORIZE_NEON_H_
    00003
    00004 /* -------------------------------------------------------------------------- *
    00005  *                                   OpenMM                                   *
    00006  * -------------------------------------------------------------------------- *
    00007  * This is part of the OpenMM molecular simulation toolkit originating from   *
    00008  * Simbios, the NIH National Center for Physics-Based Simulation of           *
    00009  * Biological Structures at Stanford, funded under the NIH Roadmap for        *
    00010  * Medical Research, grant U54 GM072970. See https://simtk.org.               *
    00011  *                                                                            *
    00012  * Portions copyright (c) 2013-2014 Stanford University and the Authors.      *
    00013  * Authors: Mateus Lima, Peter Eastman                                        *
    00014  * Contributors:                                                              *
    00015  *                                                                            *
    00016  * Permission is hereby granted, free of charge, to any person obtaining a    *
    00017  * copy of this software and associated documentation files (the "Software"), *
    00018  * to deal in the Software without restriction, including without limitation  *
    00019  * the rights to use, copy, modify, merge, publish, distribute, sublicense,   *
    00020  * and/or sell copies of the Software, and to permit persons to whom the      *
    00021  * Software is furnished to do so, subject to the following conditions:       *
    00022  *                                                                            *
    00023  * The above copyright notice and this permission notice shall be included in *
    00024  * all copies or substantial portions of the Software.                        *
    00025  *                                                                            *
    00026  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR *
    00027  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,   *
    00028  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL    *
    00029  * THE AUTHORS, CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,    *
    00030  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR      *
    00031  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE  *
    00032  * USE OR OTHER DEALINGS IN THE SOFTWARE.                                     *
    00033  * -------------------------------------------------------------------------- */
    00034
    00035 #include <cpu-features.h>
    00036 #include <arm_neon.h>
    00037 #include <cmath>
    00038
    00039 typedef int int32_t;
    00040
    00041 // This file defines classes and functions to simplify vectorizing code with NEON.
    00042
    00046 static bool isVec4Supported() {
    00047     uint64_t features = android_getCpuFeatures();
    00048     return (features & ANDROID_CPU_ARM_FEATURE_NEON) != 0;
    00049 }
    00050
    00051 class ivec4;
    00052
    00056 class fvec4 {
    00057 public:
    00058     float32x4_t val;
    00059
    00060     fvec4() {}
    00061     fvec4(float v) : val(vdupq_n_f32(v)) {}
    00062     fvec4(float v1, float v2, float v3, float v4) {
    00063         float v[] = {v1, v2, v3, v4};
    00064         val = vld1q_f32(v);
    00065     }
    00066     fvec4(float32x4_t v) : val(v) {}
    00067     fvec4(const float* v) : val(vld1q_f32(v)) {}
    00068     operator float32x4_t() const {
    00069         return val;
    00070     }
    00071     float operator[](int i) const {
    00072         switch (i) {
    00073             case 0:
    00074                 return vgetq_lane_f32(val, 0);
    00075             case 1:
    00076                 return vgetq_lane_f32(val, 1);
    00077             case 2:
    00078                 return vgetq_lane_f32(val, 2);
    00079             case 3:
    00080                 return vgetq_lane_f32(val, 3);
    00081         }
    00082         return 0.0f;
    00083     }
    00084     void store(float* v) const {
    00085         vst1q_f32(v, val);
    00086     }
    00087     fvec4 operator+(const fvec4& other) const {
    00088         return vaddq_f32(val, other);
    00089     }
    00090     fvec4 operator-(const fvec4& other) const {
    00091         return vsubq_f32(val, other);
    00092     }
    00093     fvec4 operator*(const fvec4& other) const {
    00094         return vmulq_f32(val, other);
    00095     }
    00096     fvec4 operator/(const fvec4& other) const {
    00097         // NEON does not have a divide float-point operator, so we get the reciprocal and multiply.
    00098
    00099         float32x4_t reciprocal = vrecpeq_f32(other);
    00100         reciprocal = vmulq_f32(vrecpsq_f32(other, reciprocal), reciprocal);
    00101         reciprocal = vmulq_f32(vrecpsq_f32(other, reciprocal), reciprocal);
    00102         fvec4 result = vmulq_f32(val,reciprocal);
    00103         return result;
    00104     }
    00105     void operator+=(const fvec4& other) {
    00106         val = vaddq_f32(val, other);
    00107     }
    00108     void operator-=(const fvec4& other) {
    00109         val = vsubq_f32(val, other);
    00110     }
    00111     void operator*=(const fvec4& other) {
    00112         val = vmulq_f32(val, other);
    00113     }
    00114     void operator/=(const fvec4& other) {
    00115         val = *this/other;
    00116     }
    00117     fvec4 operator-() const {
    00118         return vnegq_f32(val);
    00119     }
    00120     fvec4 operator&(const fvec4& other) const {
    00121         return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(val), vreinterpretq_u32_f32(other)));
    00122     }
    00123     fvec4 operator|(const fvec4& other) const {
    00124         return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(val), vreinterpretq_u32_f32(other)));
    00125     }
    00126     fvec4 operator==(const fvec4& other) const {
    00127         return vcvtq_f32_s32(vreinterpretq_s32_u32(vceqq_f32(val, other)));
    00128     }
    00129     fvec4 operator!=(const fvec4& other) const {
    00130         return vcvtq_f32_s32(vreinterpretq_s32_u32(vmvnq_u32(vceqq_f32(val, other)))); // not(equals(val, other))
    00131     }
    00132     fvec4 operator>(const fvec4& other) const {
    00133         return vcvtq_f32_s32(vreinterpretq_s32_u32(vcgtq_f32(val, other)));
    00134     }
    00135     fvec4 operator<(const fvec4& other) const {
    00136         return vcvtq_f32_s32(vreinterpretq_s32_u32(vcltq_f32(val, other)));
    00137     }
    00138     fvec4 operator>=(const fvec4& other) const {
    00139         return vcvtq_f32_s32(vreinterpretq_s32_u32(vcgeq_f32(val, other)));
    00140     }
    00141     fvec4 operator<=(const fvec4& other) const {
    00142         return vcvtq_f32_s32(vreinterpretq_s32_u32(vcleq_f32(val, other)));
    00143     }
    00144     operator ivec4() const;
    00145 };
    00146
    00150 class ivec4 {
    00151 public:
    00152
    00153     int32x4_t val;
    00154
    00155     ivec4() {}
    00156     ivec4(int v) : val(vdupq_n_s32(v)) {}
    00157     ivec4(int v1, int v2, int v3, int v4) {
    00158         int v[] = {v1, v2, v3, v4};
    00159         val = vld1q_s32(v);
    00160     }
    00161     ivec4(int32x4_t v) : val(v) {}
    00162     ivec4(const int* v) : val(vld1q_s32(v)) {}
    00163     operator int32x4_t() const {
    00164         return val;
    00165     }
    00166     int operator[](int i) const {
    00167         switch (i) {
    00168             case 0:
    00169                 return vgetq_lane_s32(val, 0);
    00170             case 1:
    00171                 return vgetq_lane_s32(val, 1);
    00172             case 2:
    00173                 return vgetq_lane_s32(val, 2);
    00174             case 3:
    00175                 return vgetq_lane_s32(val, 3);
    00176         }
    00177         return 0;
    00178     }
    00179     void store(int* v) const {
    00180         vst1q_s32(v, val);
    00181     }
    00182     ivec4 operator+(const ivec4& other) const {
    00183         return vaddq_s32(val, other);
    00184     }
    00185     ivec4 operator-(const ivec4& other) const {
    00186         return vsubq_s32(val, other);
    00187     }
    00188     ivec4 operator*(const ivec4& other) const {
    00189         return vmulq_s32(val, other);
    00190     }
    00191     void operator+=(const ivec4& other) {
    00192         val = vaddq_s32(val, other);
    00193     }
    00194     void operator-=(const ivec4& other) {
    00195         val = vsubq_s32(val, other);
    00196     }
    00197     void operator*=(const ivec4& other) {
    00198         val = vmulq_s32(val, other);
    00199     }
    00200     ivec4 operator-() const {
    00201         return vnegq_s32(val);
    00202     }
    00203     ivec4 operator&(const ivec4& other) const {
    00204         return vandq_s32(val, other);
    00205     }
    00206     ivec4 operator|(const ivec4& other) const {
    00207         return vorrq_s32(val, other);
    00208     }
    00209     ivec4 operator==(const ivec4& other) const {
    00210         return vreinterpretq_s32_u32(vceqq_s32(val, other));
    00211     }
    00212     ivec4 operator!=(const ivec4& other) const {
    00213         return vreinterpretq_s32_u32(vmvnq_u32(vceqq_s32(val, other))); // not(equal(val, other))
    00214     }
    00215     ivec4 operator>(const ivec4& other) const {
    00216         return vreinterpretq_s32_u32(vcgtq_s32(val, other));
    00217     }
    00218     ivec4 operator<(const ivec4& other) const {
    00219         return vreinterpretq_s32_u32(vcltq_s32(val, other));
    00220     }
    00221     ivec4 operator>=(const ivec4& other) const {
    00222         return vreinterpretq_s32_u32(vcgeq_s32(val, other));
    00223     }
    00224     ivec4 operator<=(const ivec4& other) const {
    00225         return vreinterpretq_s32_u32(vcleq_s32(val, other));
    00226     }
    00227     operator fvec4() const;
    00228 };
    00229
    00230 // Conversion operators.
    00231
    00232 inline fvec4::operator ivec4() const {
    00233     return ivec4(vcvtq_s32_f32(val));
    00234 }
    00235
    00236 inline ivec4::operator fvec4() const {
    00237     return fvec4(vcvtq_f32_s32(val));
    00238 }
    00239
    00240 // Functions that operate on fvec4s.
    00241
    00242 static inline fvec4 min(const fvec4& v1, const fvec4& v2) {
    00243     return vminq_f32(v1, v2);
    00244 }
    00245
    00246 static inline fvec4 max(const fvec4& v1, const fvec4& v2) {
    00247     return vmaxq_f32(v1, v2);
    00248 }
    00249
    00250 static inline fvec4 abs(const fvec4& v) {
    00251     return vabsq_f32(v);
    00252 }
    00253
    00254 static inline fvec4 sqrt(const fvec4& v) {
    00255     float32x4_t recipSqrt = vrsqrteq_f32(v);
    00256     recipSqrt = vmulq_f32(recipSqrt, vrsqrtsq_f32(vmulq_f32(recipSqrt, v), recipSqrt));
    00257     recipSqrt = vmulq_f32(recipSqrt, vrsqrtsq_f32(vmulq_f32(recipSqrt, v), recipSqrt));
    00258     return vmulq_f32(v, recipSqrt);
    00259 }
    00260
    00261 static inline float dot3(const fvec4& v1, const fvec4& v2) {
    00262     fvec4 result = v1*v2;
    00263     return vgetq_lane_f32(result, 0) + vgetq_lane_f32(result, 1) + vgetq_lane_f32(result, 2);
    00264 }
    00265
    00266 static inline float dot4(const fvec4& v1, const fvec4& v2) {
    00267     fvec4 result = v1*v2;
    00268     return vgetq_lane_f32(result, 0) + vgetq_lane_f32(result, 1) + vgetq_lane_f32(result, 2) + vgetq_lane_f32(result,3);
    00269 }
    00270
    00271 static inline fvec4 cross(const fvec4& v1, const fvec4& v2) {
    00272     return fvec4(v1[1]*v2[2] - v1[2]*v2[1],
    00273                  v1[2]*v2[0] - v1[0]*v2[2],
    00274                  v1[0]*v2[1] - v1[1]*v2[0], 0);
    00275 }
    00276
    00277 static inline void transpose(fvec4& v1, fvec4& v2, fvec4& v3, fvec4& v4) {
    00278     float32x4x2_t t1 = vuzpq_f32(v1, v3);
    00279     float32x4x2_t t2 = vuzpq_f32(v2, v4);
    00280     float32x4x2_t t3 = vtrnq_f32(t1.val[0], t2.val[0]);
    00281     float32x4x2_t t4 = vtrnq_f32(t1.val[1], t2.val[1]);
    00282     v1 = t3.val[0];
    00283     v2 = t4.val[0];
    00284     v3 = t3.val[1];
    00285     v4 = t4.val[1];
    00286 }
    00287
    00288 // Functions that operate on ivec4s.
    00289
    00290 static inline ivec4 min(const ivec4& v1, const ivec4& v2) {
    00291     return vminq_s32(v1, v2);
    00292 }
    00293
    00294 static inline ivec4 max(const ivec4& v1, const ivec4& v2) {
    00295     return vmaxq_s32(v1, v2);
    00296 }
    00297
    00298 static inline ivec4 abs(const ivec4& v) {
    00299     return vabdq_s32(v, ivec4(0));
    00300 }
    00301
    00302 static inline bool any(const ivec4& v) {
    00303     return (vgetq_lane_s32(v, 0) != 0 || vgetq_lane_s32(v, 1) != 0 || vgetq_lane_s32(v, 2) != 0 || vgetq_lane_s32(v, 3) != 0);
    00304 }
    00305
    00306 // Mathematical operators involving a scalar and a vector.
    00307
    00308 static inline fvec4 operator+(float v1, const fvec4& v2) {
    00309     return fvec4(v1)+v2;
    00310 }
    00311
    00312 static inline fvec4 operator-(float v1, const fvec4& v2) {
    00313     return fvec4(v1)-v2;
    00314 }
    00315
    00316 static inline fvec4 operator*(float v1, const fvec4& v2) {
    00317     return fvec4(v1)*v2;
    00318 }
    00319
    00320 static inline fvec4 operator/(float v1, const fvec4& v2) {
    00321     return fvec4(v1)/v2;
    00322 }
    00323
    00324 // Operations for blending fvec4s based on an ivec4.
    00325
    00326 static inline fvec4 blend(const fvec4& v1, const fvec4& v2, const ivec4& mask) {
    00327     return vbslq_f32(vreinterpretq_u32_s32(mask), v2, v1);
    00328 }
    00329
    00330 // These are at the end since they involve other functions defined above.
    00331
    00332 static inline fvec4 round(const fvec4& v) {
    00333     fvec4 shift(0x1.0p23f);
    00334     fvec4 absResult = (abs(v)+shift)-shift;
    00335     return blend(v, absResult, ivec4(0x7FFFFFFF));
    00336 }
    00337
    00338 static inline fvec4 floor(const fvec4& v) {
    00339     fvec4 rounded = round(v);
    00340     return rounded + blend(0.0f, -1.0f, rounded>v);
    00341 }
    00342
    00343 static inline fvec4 ceil(const fvec4& v) {
    00344     fvec4 rounded = round(v);
    00345     return rounded + blend(0.0f, 1.0f, rounded<v);
    00346 }
    00347
    00348 #endif /*OPENMM_VECTORIZE_NEON_H_*/