    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.reflect.Array;
    017import java.util.ArrayList;
    018import java.util.Arrays;
    019import java.util.Comparator;
    020import java.util.List;
    021
    022import org.apache.commons.math3.util.MathArrays;
    023import org.eclipse.january.DatasetException;
    024import org.slf4j.Logger;
    025import org.slf4j.LoggerFactory;
    026
    027/**
    028 * Utilities for manipulating datasets
    029 */
    030@SuppressWarnings("unchecked")
    031public class DatasetUtils {
    032
    033        /**
    034         * Setup the logging facilities
    035         */
    036        transient protected static final Logger utilsLogger = LoggerFactory.getLogger(DatasetUtils.class);
    037
    038        /**
    039         * Append copy of dataset with another dataset along n-th axis
    040         *
    041         * @param a
    042         * @param b
    043         * @param axis
    044         *            number of axis (negative number counts from last)
    045         * @return appended dataset
    046         */
    047        public static Dataset append(IDataset a, IDataset b, int axis) {
    048                final int[] ashape = a.getShape();
    049                final int rank = ashape.length;
    050                final int[] bshape = b.getShape();
    051                if (rank != bshape.length) {
    052                        throw new IllegalArgumentException("Incompatible number of dimensions");
    053                }
    054                axis = ShapeUtils.checkAxis(rank, axis);
    055
    056                for (int i = 0; i < rank; i++) {
    057                        if (i != axis && ashape[i] != bshape[i]) {
    058                                throw new IllegalArgumentException("Incompatible dimensions");
    059                        }
    060                }
    061                final int[] nshape = new int[rank];
    062                for (int i = 0; i < rank; i++) {
    063                        nshape[i] = ashape[i];
    064                }
    065                nshape[axis] += bshape[axis];
    066                final Class<? extends Dataset> ot = InterfaceUtils.getInterface(b);
    067                final Class<? extends Dataset> dt = InterfaceUtils.getInterface(a);
    068                Dataset ds = DatasetFactory.zeros(a.getElementsPerItem(), InterfaceUtils.getBestInterface(ot, dt), nshape);
    069                IndexIterator iter = ds.getIterator(true);
    070                int[] pos = iter.getPos();
    071                while (iter.hasNext()) {
    072                        int d = ashape[axis];
    073                        if (pos[axis] < d) {
    074                                ds.setObjectAbs(iter.index, a.getObject(pos));
    075                        } else {
    076                                pos[axis] -= d;
    077                                ds.setObjectAbs(iter.index, b.getObject(pos));
    078                                pos[axis] += d;
    079                        }
    080                }
    081
    082                return ds;
    083        }
    084
    085        /**
    086         * Changes specific items of dataset by replacing them with other array
    087         * @param a
    088         * @param indices dataset interpreted as integers
    089         * @param values
    090         * @return changed dataset
    091         */
    092        public static <T extends Dataset> T put(final T a, final Dataset indices, Object values) {
    093                IndexIterator it = indices.getIterator();
    094                Dataset vd = DatasetFactory.createFromObject(values).flatten();
    095                int vlen = vd.getSize();
    096                int v = 0;
    097                while (it.hasNext()) {
    098                        if (v >= vlen) v -= vlen;
    099
    100                        a.setObjectAbs((int) indices.getElementLongAbs(it.index), vd.getObjectAbs(v++));
    101                }
    102                return a;
    103        }
    104
    105        /**
    106         * Changes specific items of dataset by replacing them with other array
    107         * @param a
    108         * @param indices
    109         * @param values
    110         * @return changed dataset
    111         */
    112        public static <T extends Dataset> T put(final T a, final int[] indices, Object values) {
    113                int ilen = indices.length;
    114                Dataset vd = DatasetFactory.createFromObject(values).flatten();
    115                int vlen = vd.getSize();
    116                for (int i = 0, v= 0; i < ilen; i++) {
    117                        if (v >= vlen) v -= vlen;
    118
    119                        a.setObjectAbs(indices[i], vd.getObjectAbs(v++));
    120                }
    121                return a;
    122        }
    123
    124        /**
    125         * Take items from dataset along an axis
    126         * @param indices dataset interpreted as integers
    127         * @param axis if null, then use flattened view
    128         * @return a sub-array
    129         */
    130        public static <T extends Dataset> T take(final T a, final Dataset indices, Integer axis) {
    131                IntegerDataset indexes = (IntegerDataset) indices.flatten().cast(IntegerDataset.class);
    132                return take(a, indexes.getData(), axis);
    133        }
    134
    135        /**
    136         * Take items from dataset along an axis
    137         * @param indices
    138         * @param axis if null, then use flattened view
    139         * @return a sub-array
    140         */
    141        public static <T extends Dataset> T take(final T a, final int[] indices, Integer axis) {
    142                if (indices == null || indices.length == 0) {
    143                        utilsLogger.error("No indices given");
    144                        throw new IllegalArgumentException("No indices given");
    145                }
    146                int[] ashape = a.getShape();
    147                final int rank = ashape.length;
    148                final Class<? extends Dataset> ac = a.getClass();
    149                final int ilen = indices.length;
    150                final int is = a.getElementsPerItem();
    151
    152                Dataset result;
    153                if (axis == null) {
    154                        ashape = new int[1];
    155                        ashape[0] = ilen;
    156                        result = DatasetFactory.zeros(is, ac, ashape);
    157                        Serializable src = a.getBuffer();
    158                        for (int i = 0; i < ilen; i++) {
    159                                ((AbstractDataset) result).setItemDirect(i, indices[i], src);
    160                        }
    161                } else {
    162                        axis = a.checkAxis(axis);
    163                        ashape[axis] = ilen;
    164                        result = DatasetFactory.zeros(is, ac, ashape);
    165
    166                        int[] dpos = new int[rank];
    167                        int[] spos = new int[rank];
    168                        boolean[] axes = new boolean[rank];
    169                        Arrays.fill(axes, true);
    170                        axes[axis] = false;
    171                        Serializable src = a.getBuffer();
    172                        for (int i = 0; i < ilen; i++) {
    173                                spos[axis] = indices[i];
    174                                dpos[axis] = i;
    175                                SliceIterator siter = a.getSliceIteratorFromAxes(spos, axes);
    176                                SliceIterator diter = result.getSliceIteratorFromAxes(dpos, axes);
    177
    178                                while (siter.hasNext() && diter.hasNext()) {
    179                                        ((AbstractDataset) result).setItemDirect(diter.index, siter.index, src);
    180                                }
    181                        }
    182                }
    183                result.setDirty();
    184                return (T) result;
    185        }
    186
    187        /**
    188         * Construct a dataset that contains the original dataset repeated the number
    189         * of times in each axis given by corresponding entries in the reps array
    190         *
    191         * @param a
    192         * @param reps
    193         * @return tiled dataset
    194         */
    195        public static Dataset tile(final IDataset a, int... reps) {
    196                int[] shape = a.getShape();
    197                int rank = shape.length;
    198                final int rlen = reps.length;
    199
    200                // expand shape
    201                if (rank < rlen) {
    202                        int[] newShape = new int[rlen];
    203                        int extraRank = rlen - rank;
    204                        for (int i = 0; i < extraRank; i++) {
    205                                newShape[i] = 1;
    206                        }
    207                        for (int i = 0; i < rank; i++) {
    208                                newShape[i+extraRank] = shape[i];
    209                        }
    210
    211                        shape = newShape;
    212                        rank = rlen;
    213                } else if (rank > rlen) {
    214                        int[] newReps = new int[rank];
    215                        int extraRank = rank - rlen;
    216                        for (int i = 0; i < extraRank; i++) {
    217                                newReps[i] = 1;
    218                        }
    219                        for (int i = 0; i < rlen; i++) {
    220                                newReps[i+extraRank] = reps[i];
    221                        }
    222                        reps = newReps;
    223                }
    224
    225                // calculate new shape
    226                int[] newShape = new int[rank];
    227                for (int i = 0; i < rank; i++) {
    228                        newShape[i] = shape[i]*reps[i];
    229                }
    230
    231                Dataset tdata = DatasetFactory.zeros(a.getElementsPerItem(), InterfaceUtils.getInterfaceFromClass(a.getElementsPerItem(), a.getElementClass()), newShape);
    232
    233                // decide which way to put slices
    234                boolean manyColumns;
    235                if (rank == 1)
    236                        manyColumns = true;
    237                else
    238                        manyColumns = shape[rank-1] > 64;
    239
    240                if (manyColumns) {
    241                        // generate each start point and put a slice in
    242                        IndexIterator iter = tdata.getSliceIterator(null, null, shape);
    243                        SliceIterator siter = (SliceIterator) tdata.getSliceIterator(null, shape, null);
    244                        final int[] pos = iter.getPos();
    245                        while (iter.hasNext()) {
    246                                siter.setStart(pos);
    247                                tdata.setSlice(a, siter);
    248                        }
    249
    250                } else {
    251                        // for each value, set slice given by repeats
    252                        final int[] skip = new int[rank];
    253                        for (int i = 0; i < rank; i++) {
    254                                if (reps[i] == 1) {
    255                                        skip[i] = newShape[i];
    256                                } else {
    257                                        skip[i] = shape[i];
    258                                }
    259                        }
    260
    261                        Dataset aa = convertToDataset(a);
    262                        IndexIterator ita = aa.getIterator(true);
    263                        final int[] pos = ita.getPos();
    264
    265                        final int[] sstart = new int[rank];
    266                        final int extra = rank - pos.length;
    267                        for (int i = 0; i < extra; i++) {
    268                                sstart[i] = 0;
    269                        }
    270                        SliceIterator siter = (SliceIterator) tdata.getSliceIterator(sstart, null, skip);
    271                        while (ita.hasNext()) {
    272                                for (int i = 0; i < pos.length; i++) {
    273                                        sstart[i + extra] = pos[i];
    274                                }
    275                                siter.setStart(sstart);
    276                                tdata.setSlice(aa.getObjectAbs(ita.index), siter);
    277                        }
    278                }
    279
    280                return tdata;
    281        }
    282
    283        /**
    284         * Permute copy of dataset's axes so that given order is old order:
    285         * <pre>{@literal
    286         *  axisPerm = (p(0), p(1),...) => newdata(n(0), n(1),...) = olddata(o(0), o(1), ...)
    287         *  such that n(i) = o(p(i)) for all i
    288         * }</pre>
    289         * I.e. for a 3D dataset (1,0,2) implies the new dataset has its 1st dimension
    290         * running along the old dataset's 2nd dimension and the new 2nd is the old 1st.
    291         * The 3rd dimension is left unchanged.
    292         *
    293         * @param a
    294         * @param axes if null or zero length then axes order reversed
    295         * @return remapped copy of data
    296         */
    297        public static Dataset transpose(final IDataset a, int... axes) {
    298                return convertToDataset(a).transpose(axes);
    299        }
    300
    301        /**
    302         * Swap two axes in dataset
    303         * @param a
    304         * @param axis1
    305         * @param axis2
    306         * @return swapped dataset
    307         */
    308        public static Dataset swapAxes(final IDataset a, int axis1, int axis2) {
    309                return convertToDataset(a).swapAxes(axis1, axis2);
    310        }
    311
    312        /**
    313         * @param a
    314         * @return sorted flattened copy of dataset
    315         */
    316        public static <T extends Dataset> T sort(final T a) {
    317                return sort(a, (Integer) null);
    318        }
    319
    320        /**
    321         * @param a
    322         * @param axis to sort along, if null then dataset is first flattened
    323         * @return dataset sorted along axis
    324         */
    325        public static <T extends Dataset> T sort(final T a, final Integer axis) {
    326                Dataset s = a.clone();
    327                return (T) s.sort(axis);
    328        }
    329
    330        /**
    331         * Sort in place given dataset and reorder ancillary datasets too
    332         * @param a dataset to be sorted
    333         * @param b ancillary datasets
    334         */
    335        public static void sort(Dataset a, Dataset... b) {
    336                if (!DTypeUtils.isDTypeNumerical(a.getDType())) {
    337                        throw new UnsupportedOperationException("Sorting non-numerical datasets not supported yet");
    338                }
    339
    340                // gather all datasets as double dataset copies
    341                DoubleDataset s = copy(DoubleDataset.class, a);
    342                int l = b == null ? 0 : b.length;
    343                DoubleDataset[] t = new DoubleDataset[l];
    344                int n = 0;
    345                for (int i = 0; i < l; i++) {
    346                        if (b[i] != null) {
    347                                if (!DTypeUtils.isDTypeNumerical(b[i].getDType())) {
    348                                        throw new UnsupportedOperationException("Sorting non-numerical datasets not supported yet");
    349                                }
    350                                t[i] = copy(DoubleDataset.class, b[i]);
    351                                n++;
    352                        }
    353                }
    354
    355                double[][] y = new double[n][];
    356                for (int i = 0, j = 0; i < l; i++) {
    357                        if (t[i] != null) {
    358                                y[j++] = t[i].getData();
    359                        }
    360                }
    361
    362                MathArrays.sortInPlace(s.getData(), y);
    363
    364                a.setSlice(s);
    365                for (int i = 0; i < l; i++) {
    366                        if (b[i] != null) {
    367                                b[i].setSlice(t[i]);
    368                        }
    369                }
    370        }
    371
    372        /**
    373         * Indirectly sort along given axis
    374         * @param a dataset whose indexes will be sorted
    375         * @param axis to sort along, if null then dataset is first flattened
    376         * @return indexes
    377         * @since 2.1
    378         */
    379        public static IntegerDataset indexSort(Dataset a, Integer axis) {
    380                if (axis == null) {
    381                        int size = a.getSize();
    382                        Integer[] index = new Integer[size];
    383                        for (int i = 0; i < size; i++) {
    384                                index[i] = i;
    385                        }
    386                        final Dataset f = a.flatten(); // is this correct for views??? Check with NumPy
    387                        Comparator<Integer> cmp = new Comparator<Integer>() {
    388
    389                                @Override
    390                                public int compare(Integer o1, Integer o2) {
    391
    392                                        return Double.compare(f.getElementDoubleAbs(o1), f.getElementDoubleAbs(o2));
    393                                }
    394                        };
    395                        Arrays.sort(index, cmp);
    396                        return DatasetFactory.createFromObject(IntegerDataset.class, index);
    397                }
    398
    399                axis = a.checkAxis(axis);
    400                final int[] shape = a.getShapeRef();
    401                IntegerDataset id = DatasetFactory.zeros(IntegerDataset.class, shape);
    402                int size = shape[axis];
    403                Integer[] index = new Integer[size];
    404
    405                int[] dShape = new int[shape.length];
    406                Arrays.fill(dShape, 1);
    407                dShape[axis] = size;
    408                final DoubleDataset dd = DatasetFactory.zeros(DoubleDataset.class, dShape);
    409                final Comparator<Integer> cmp = new Comparator<Integer>() {
    410                        @Override
    411                        public int compare(Integer o1, Integer o2) {
    412
    413                                return Double.compare(dd.getElementDoubleAbs(o1), dd.getElementDoubleAbs(o2));
    414                        }
    415                };
    416
    417                SliceND ds = new SliceND(dShape);
    418                SliceNDIterator it = new SliceNDIterator(new SliceND(shape), axis);
    419                int[] pos = it.getPos();
    420                int[] ipos = pos.clone();
    421                while (it.hasNext()) {
    422                        dd.setSlice(a.getSliceView(it.getCurrentSlice()), ds);
    423                        for (int i = 0; i < size; i++) {
    424                                index[i] = i;
    425                        }
    426                        Arrays.sort(index, cmp);
    427
    428                        System.arraycopy(pos, 0, ipos, 0, pos.length);
    429                        for (int i = 0; i < size; i++) {
    430                                ipos[axis] = i;
    431                                id.set(index[i], ipos);
    432                        }
    433                }
    434
    435                return id;
    436        }
    437
    438        /**
    439         * Concatenate the set of datasets along given axis
    440         * @param as
    441         * @param axis
    442         * @return concatenated dataset
    443         */
    444        public static Dataset concatenate(final IDataset[] as, int axis) {
    445                if (as == null || as.length == 0) {
    446                        utilsLogger.error("No datasets given");
    447                        throw new IllegalArgumentException("No datasets given");
    448                }
    449                IDataset a = as[0];
    450                if (as.length == 1) {
    451                        return convertToDataset(a.clone());
    452                }
    453
    454                int[] ashape = a.getShape();
    455                axis = ShapeUtils.checkAxis(ashape.length, axis);
    456                Class<? extends Dataset> ac = InterfaceUtils.getInterface(a);
    457                int anum = as.length;
    458                int isize = a.getElementsPerItem();
    459
    460                int i = 1;
    461                for (; i < anum; i++) {
    462                        if (!ac.equals(InterfaceUtils.getInterface(as[i]))) {
    463                                utilsLogger.error("Datasets are not of same type");
    464                                break;
    465                        }
    466                        if (!ShapeUtils.areShapesCompatible(ashape, as[i].getShape(), axis)) {
    467                                utilsLogger.error("Datasets' shapes are not equal");
    468                                break;
    469                        }
    470                        final int is = as[i].getElementsPerItem();
    471                        if (isize < is)
    472                                isize = is;
    473                }
    474                if (i < anum) {
    475                        utilsLogger.error("Dataset are not compatible");
    476                        throw new IllegalArgumentException("Datasets are not compatible");
    477                }
    478
    479                for (i = 1; i < anum; i++) {
    480                        ashape[axis] += as[i].getShape()[axis];
    481                }
    482
    483                Dataset result = DatasetFactory.zeros(isize, ac, ashape);
    484
    485                int[] start = new int[ashape.length];
    486                int[] stop = ashape;
    487                stop[axis] = 0;
    488                for (i = 0; i < anum; i++) {
    489                        IDataset b = as[i];
    490                        int[] bshape = b.getShape();
    491                        stop[axis] += bshape[axis];
    492                        result.setSlice(b, start, stop, null);
    493                        start[axis] += bshape[axis];
    494                }
    495
    496                return result;
    497        }
    498
    499        /**
    500         * Split a dataset into equal sections along given axis
    501         * @param a
    502         * @param sections
    503         * @param axis
    504         * @param checkEqual makes sure the division is into equal parts
    505         * @return list of split datasets
    506         */
    507        public static List<Dataset> split(final Dataset a, int sections, int axis, final boolean checkEqual) {
    508                int[] ashape = a.getShapeRef();
    509                axis = a.checkAxis(axis);
    510                int imax = ashape[axis];
    511                if (checkEqual && (imax%sections) != 0) {
    512                        utilsLogger.error("Number of sections does not divide axis into equal parts");
    513                        throw new IllegalArgumentException("Number of sections does not divide axis into equal parts");
    514                }
    515                int n = (imax + sections - 1) / sections;
    516                int[] indices = new int[sections-1];
    517                for (int i = 1; i < sections; i++)
    518                        indices[i-1] = n*i;
    519                return split(a, indices, axis);
    520        }
    521
    522        /**
    523         * Split a dataset into parts along given axis
    524         * @param a
    525         * @param indices
    526         * @param axis
    527         * @return list of split datasets
    528         */
    529        public static List<Dataset> split(final Dataset a, int[] indices, int axis) {
    530                final int[] ashape = a.getShapeRef();
    531                axis = a.checkAxis(axis);
    532                final int rank = ashape.length;
    533                final int imax = ashape[axis];
    534
    535                final List<Dataset> result = new ArrayList<Dataset>();
    536
    537                final int[] nshape = ashape.clone();
    538                final int is = a.getElementsPerItem();
    539
    540                int oind = 0;
    541                final int[] start = new int[rank];
    542                final int[] stop = new int[rank];
    543                final int[] step = new int[rank];
    544                for (int i = 0; i < rank; i++) {
    545                        start[i] = 0;
    546                        stop[i] = ashape[i];
    547                        step[i] = 1;
    548                }
    549                for (int ind : indices) {
    550                        if (ind > imax) {
    551                                result.add(DatasetFactory.zeros(is, a.getClass(), 0));
    552                        } else {
    553                                nshape[axis] = ind - oind;
    554                                start[axis] = oind;
    555                                stop[axis] = ind;
    556                                Dataset n = DatasetFactory.zeros(is, a.getClass(), nshape);
    557                                IndexIterator iter = a.getSliceIterator(start, stop, step);
    558
    559                                a.fillDataset(n, iter);
    560                                result.add(n);
    561                                oind = ind;
    562                        }
    563                }
    564
    565                if (imax > oind) {
    566                        nshape[axis] = imax - oind;
    567                        start[axis] = oind;
    568                        stop[axis] = imax;
    569                        Dataset n = DatasetFactory.zeros(is, a.getClass(), nshape);
    570                        IndexIterator iter = a.getSliceIterator(start, stop, step);
    571
    572                        a.fillDataset(n, iter);
    573                        result.add(n);
    574                }
    575
    576                return result;
    577        }
    578
    579        /**
    580         * Constructs a dataset which has its elements along an axis replicated from
    581         * the original dataset by the number of times given in the repeats array.
    582         *
    583         * By default, axis=-1 implies using a flattened version of the input dataset
    584         *
    585         * @param a
    586         * @param repeats
    587         * @param axis
    588         * @return dataset
    589         */
    590        public static <T extends Dataset> T repeat(T a, int[] repeats, int axis) {
    591                Serializable buf = a.getBuffer();
    592                int[] shape = a.getShape();
    593                int rank = shape.length;
    594                final int is = a.getElementsPerItem();
    595
    596                if (axis >= rank) {
    597                        utilsLogger.warn("Axis value is out of bounds");
    598                        throw new IllegalArgumentException("Axis value is out of bounds");
    599                }
    600
    601                int alen;
    602                if (axis < 0) {
    603                        alen = a.getSize();
    604                        axis = 0;
    605                        rank = 1;
    606                        shape[0] = alen;
    607                } else {
    608                        alen = shape[axis];
    609                }
    610                int rlen = repeats.length;
    611                if (rlen != 1 && rlen != alen) {
    612                        utilsLogger.warn("Repeats array should have length of 1 or match chosen axis");
    613                        throw new IllegalArgumentException("Repeats array should have length of 1 or match chosen axis");
    614                }
    615
    616                for (int i = 0; i < rlen; i++) {
    617                        if (repeats[i] < 0) {
    618                                utilsLogger.warn("Negative repeat value is not allowed");
    619                                throw new IllegalArgumentException("Negative repeat value is not allowed");
    620                        }
    621                }
    622
    623                int[] newShape = new int[rank];
    624                for (int i = 0; i < rank; i ++)
    625                        newShape[i] = shape[i];
    626
    627                // do single repeat separately
    628                if (repeats.length == 1) {
    629                        newShape[axis] *= repeats[0];
    630                } else {
    631                        int nlen = 0;
    632                        for (int i = 0; i < alen; i++) {
    633                                nlen += repeats[i];
    634                        }
    635                        newShape[axis] = nlen;
    636                }
    637
    638                Dataset rdata = DatasetFactory.zeros(is, a.getClass(), newShape);
    639                Serializable nbuf = rdata.getBuffer();
    640
    641                int csize = is; // chunk size
    642                for (int i = axis+1; i < rank; i++) {
    643                        csize *= newShape[i];
    644                }
    645                int nout = 1;
    646                for (int i = 0; i < axis; i++) {
    647                        nout *= newShape[i];
    648                }
    649
    650                int oi = 0;
    651                int ni = 0;
    652                if (rlen == 1) { // do single repeat separately
    653                        for (int i = 0; i < nout; i++) {
    654                                for (int j = 0; j < shape[axis]; j++) {
    655                                        for (int k = 0; k < repeats[0]; k++) {
    656                                                System.arraycopy(buf, oi, nbuf, ni, csize);
    657                                                ni += csize;
    658                                        }
    659                                        oi += csize;
    660                                }
    661                        }
    662                } else {
    663                        for (int i = 0; i < nout; i++) {
    664                                for (int j = 0; j < shape[axis]; j++) {
    665                                        for (int k = 0; k < repeats[j]; k++) {
    666                                                System.arraycopy(buf, oi, nbuf, ni, csize);
    667                                                ni += csize;
    668                                        }
    669                                        oi += csize;
    670                                }
    671                        }
    672                }
    673
    674                return (T) rdata;
    675        }
    676
    677        /**
    678         * Resize a dataset
    679         * @param a
    680         * @param shape
    681         * @return new dataset with new shape and items that are truncated or repeated, as necessary
    682         */
    683        public static <T extends Dataset> T resize(final T a, final int... shape) {
    684                int size = a.getSize();
    685                Dataset rdata = DatasetFactory.zeros(a.getElementsPerItem(), a.getClass(), shape);
    686                IndexIterator it = rdata.getIterator();
    687                while (it.hasNext()) {
    688                        rdata.setObjectAbs(it.index, a.getObjectAbs(it.index % size));
    689                }
    690
    691                return (T) rdata;
    692        }
    693
    694        /**
    695         * Copy and cast a dataset
    696         *
    697         * @param d
    698         *            The dataset to be copied
    699         * @param dtype dataset type
    700         * @return copied dataset of given type
    701         * @deprecated Please use the class-based methods in DatasetUtils,
    702         *             such as {@link #copy(Class, IDataset)}
    703         */
    704        @Deprecated
    705        public static Dataset copy(final IDataset d, final int dtype) {
    706                return copy(DTypeUtils.getInterface(dtype), d);
    707        }
    708
    709        /**
    710         * Cast a dataset
    711         *
    712         * @param clazz dataset class
    713         * @param d
    714         *            The dataset to be cast.
    715         * @return dataset of given class (or same dataset if already of the right class)
    716         */
    717        public static <T extends Dataset> T copy(Class<T> clazz, final IDataset d) {
    718                Dataset a = convertToDataset(d);
    719                Dataset c = null;
    720                try {
    721                        // copy across the data
    722                        if (BooleanDataset.class.equals(clazz)) {
    723                                c = new BooleanDataset(a);
    724                        } else if (ByteDataset.class.equals(clazz)) {
    725                                c = new ByteDataset(a);
    726                        } else if (ShortDataset.class.equals(clazz)) {
    727                                c = new ShortDataset(a);
    728                        } else if (IntegerDataset.class.equals(clazz)) {
    729                                c = new IntegerDataset(a);
    730                        } else if (LongDataset.class.equals(clazz)) {
    731                                c = new LongDataset(a);
    732                        } else if (CompoundByteDataset.class.equals(clazz)) {
    733                                if (a instanceof CompoundByteDataset) {
    734                                        c = new CompoundByteDataset((CompoundDataset) a);
    735                                } else {
    736                                        c = new CompoundByteDataset(a);
    737                                }
    738                        } else if (CompoundShortDataset.class.equals(clazz)) {
    739                                if (a instanceof CompoundDataset) {
    740                                        c = new CompoundShortDataset((CompoundDataset) a);
    741                                } else {
    742                                        c = new CompoundShortDataset(a);
    743                                }
    744                        } else if (CompoundIntegerDataset.class.equals(clazz)) {
    745                                if (a instanceof CompoundDataset) {
    746                                        c = new CompoundIntegerDataset((CompoundDataset) a);
    747                                } else {
    748                                        c = new CompoundIntegerDataset(a);
    749                                }
    750                        } else if (CompoundLongDataset.class.equals(clazz)) {
    751                                if (a instanceof CompoundDataset) {
    752                                        c = new CompoundLongDataset((CompoundDataset) a);
    753                                } else {
    754                                        c = new CompoundLongDataset(a);
    755                                }
    756                        } else if (FloatDataset.class.equals(clazz)) {
    757                                c = new FloatDataset(a);
    758                        } else if (DoubleDataset.class.equals(clazz)) {
    759                                c = new DoubleDataset(a);
    760                        } else if (CompoundFloatDataset.class.equals(clazz)) {
    761                                if (a instanceof CompoundDataset) {
    762                                        c = new CompoundFloatDataset((CompoundDataset) a);
    763                                } else {
    764                                        c = new CompoundFloatDataset(a);
    765                                }
    766                        } else if (CompoundDoubleDataset.class.equals(clazz)) {
    767                                if (a instanceof CompoundDataset) {
    768                                        c = new CompoundDoubleDataset((CompoundDataset) a);
    769                                } else {
    770                                        c = new CompoundDoubleDataset(a);
    771                                }
    772                        } else if (ComplexFloatDataset.class.equals(clazz)) {
    773                                c = new ComplexFloatDataset(a);
    774                        } else if (ComplexDoubleDataset.class.equals(clazz)) {
    775                                c = new ComplexDoubleDataset(a);
    776                        } else if (RGBDataset.class.equals(clazz)) {
    777                                if (a instanceof CompoundDataset) {
    778                                        c = RGBDataset.createFromCompoundDataset((CompoundDataset) a);
    779                                } else {
    780                                        c = new RGBDataset(a);
    781                                }
    782                        } else if (StringDataset.class.equals(clazz)) {
    783                                c = new StringDataset(a);
    784                        } else if (DateDataset.class.equals(clazz)) {
    785                                c = DateDatasetImpl.createFromObject(a);
    786                        } else if (ObjectDataset.class.equals(clazz)) {
    787                                c = new ObjectDataset(a);
    788                        } else {
    789                                utilsLogger.error("Dataset of unknown type!");
    790                        }
    791                } catch (OutOfMemoryError e) {
    792                        utilsLogger.error("Not enough memory available to create dataset");
    793                        throw new OutOfMemoryError("Not enough memory available to create dataset");
    794                }
    795
    796                return (T) c;
    797        }
    798
    799        /**
    800         * Cast a dataset
    801         *
    802         * @param d
    803         *            The dataset to be cast.
    804         * @param dtype dataset type
    805         * @return dataset of given type (or same dataset if already of the right type)
    806         * @deprecated Please use the class-based methods in DatasetUtils,
    807         *             such as {@link #cast(Class, IDataset)}
    808         */
    809        @Deprecated
    810        public static Dataset cast(final IDataset d, final int dtype) {
    811                return cast(DTypeUtils.getInterface(dtype), d);
    812        }
    813
    814        /**
    815         * Cast a dataset
    816         * @param clazz dataset class
    817         * @param d
    818         *            The dataset to be cast.
    819         * @return dataset of given class
    820         */
    821        public static <T extends Dataset> T cast(Class<T> clazz, final IDataset d) {
    822                Dataset a = convertToDataset(d);
    823
    824                if ((a.getClass().equals(clazz))) {
    825                        return (T) a;
    826                }
    827                return copy(clazz, d);
    828        }
    829
    830        /**
    831         * Cast a dataset
    832         *
    833         * @param d
    834         *            The dataset to be cast.
    835         * @param repeat repeat elements over item
    836         * @param dtype dataset type
    837         * @param isize item size
    838         * @return dataset of given type
    839         * @deprecated Please use the class-based methods in DatasetUtils,
    840         *             such as {@link #cast(Class, IDataset)}
    841         */
    842        @Deprecated
    843        public static Dataset cast(final IDataset d, final boolean repeat, final int dtype, final int isize) {
    844                return cast(isize, DTypeUtils.getInterface(dtype), d, repeat);
    845        }
    846
    847        /**
    848         * Cast a dataset
    849         *
    850         * @param clazz dataset class
    851         * @param d
    852         *            The dataset to be cast.
    853         * @param repeat repeat elements over item
    854         * @param dtype dataset type
    855         * @param isize item size
    856         * @since 2.3
    857         */
    858        public static <T extends Dataset> T cast(final int isize, Class<T> clazz, final IDataset d, final boolean repeat) {
    859                Dataset a = convertToDataset(d);
    860
    861                if (a.getClass().equals(clazz) && a.getElementsPerItem() == isize) {
    862                        return (T) a;
    863                }
    864                if (isize <= 0) {
    865                        utilsLogger.error("Item size is invalid (>0)");
    866                        throw new IllegalArgumentException("Item size is invalid (>0)");
    867                }
    868                if (isize > 1 && !InterfaceUtils.isComplex(clazz) && InterfaceUtils.isElemental(clazz)) {
    869                        utilsLogger.error("Item size is inconsistent with dataset type");
    870                        throw new IllegalArgumentException("Item size is inconsistent with dataset type");
    871                }
    872
    873                Dataset c = null;
    874
    875                try {
    876                        // copy across the data
    877                        if (BooleanDataset.class.equals(clazz)) {
    878                                c = new BooleanDataset(a);
    879                        } else if (ByteDataset.class.equals(clazz)) {
    880                                c = new ByteDataset(a);
    881                        } else if (ShortDataset.class.equals(clazz)) {
    882                                c = new ShortDataset(a);
    883                        } else if (IntegerDataset.class.equals(clazz)) {
    884                                c = new IntegerDataset(a);
    885                        } else if (LongDataset.class.equals(clazz)) {
    886                                c = new LongDataset(a);
    887                        } else if (CompoundByteDataset.class.equals(clazz)) {
    888                                c = new CompoundByteDataset(isize, repeat, a);
    889                        } else if (CompoundShortDataset.class.equals(clazz)) {
    890                                c = new CompoundShortDataset(isize, repeat, a);
    891                        } else if (CompoundIntegerDataset.class.equals(clazz)) {
    892                                c = new CompoundIntegerDataset(isize, repeat, a);
    893                        } else if (CompoundLongDataset.class.equals(clazz)) {
    894                                c = new CompoundLongDataset(isize, repeat, a);
    895                        } else if (FloatDataset.class.equals(clazz)) {
    896                                c = new FloatDataset(a);
    897                        } else if (DoubleDataset.class.equals(clazz)) {
    898                                c = new DoubleDataset(a);
    899                        } else if (CompoundFloatDataset.class.equals(clazz)) {
    900                                c = new CompoundFloatDataset(isize, repeat, a);
    901                        } else if (CompoundDoubleDataset.class.equals(clazz)) {
    902                                c = new CompoundDoubleDataset(isize, repeat, a);
    903                        } else if (ComplexFloatDataset.class.equals(clazz)) {
    904                                c = new ComplexFloatDataset(a);
    905                        } else if (ComplexDoubleDataset.class.equals(clazz)) {
    906                                c = new ComplexDoubleDataset(a);
    907                        } else if (RGBDataset.class.equals(clazz)) {
    908                                if (a instanceof CompoundDataset) {
    909                                        c = RGBDataset.createFromCompoundDataset((CompoundDataset) a);
    910                                } else {
    911                                        c = new RGBDataset(a);
    912                                }
    913                        } else if (StringDataset.class.equals(clazz)) {
    914                                c = new StringDataset(a);
    915                        } else if (DateDataset.class.equals(clazz)) {
    916                                c = DateDatasetImpl.createFromObject(a);
    917                        } else if (ObjectDataset.class.equals(clazz)) {
    918                                c = new ObjectDataset(a);
    919                        } else {
    920                                utilsLogger.error("Dataset of unknown type!");
    921                        }
    922                } catch (OutOfMemoryError e) {
    923                        utilsLogger.error("Not enough memory available to create dataset");
    924                        throw new OutOfMemoryError("Not enough memory available to create dataset");
    925                }
    926
    927                return (T) c;
    928        }
    929
    930        /**
    931         * Cast array of datasets to a compound dataset
    932         *
    933         * @param clazz dataset class
    934         * @param a
    935         *            The datasets to be cast.
    936         * @return dataset of given class
    937         * @since 2.3
    938         */
    939        public static  <T extends CompoundDataset> T compoundCast(Class<T> clazz, final Dataset... a) {
    940                return (T) cast((Class<? extends Dataset>) clazz, a);
    941        }
    942
    943
    944        /**
    945         * Cast array of datasets to a compound dataset
    946         *
    947         * @param clazz dataset class
    948         * @param a
    949         *            The datasets to be cast.
    950         * @return compound dataset of given class
    951         * @since 2.3
    952         */
    953        public static  CompoundDataset cast(Class<? extends Dataset> clazz, final Dataset... a) {
    954                CompoundDataset c = null;
    955                if (ByteDataset.class.equals(clazz) || CompoundByteDataset.class.equals(clazz)) {
    956                        c = new CompoundByteDataset(a);
    957                } else if (ShortDataset.class.equals(clazz) || CompoundShortDataset.class.equals(clazz)) {
    958                        c = new CompoundShortDataset(a);
    959                } else if (IntegerDataset.class.equals(clazz) || CompoundIntegerDataset.class.equals(clazz)) {
    960                        c = new CompoundIntegerDataset(a);
    961                } else if (LongDataset.class.equals(clazz) || CompoundLongDataset.class.equals(clazz)) {
    962                        c = new CompoundLongDataset(a);
    963                } else if (FloatDataset.class.equals(clazz) || CompoundFloatDataset.class.equals(clazz)) {
    964                        c = new CompoundFloatDataset(a);
    965                } else if (DoubleDataset.class.equals(clazz) || CompoundDoubleDataset.class.equals(clazz)) {
    966                        c = new CompoundDoubleDataset(a);
    967                } else if (ComplexFloatDataset.class.equals(clazz)) {
    968                        if (a.length != 2) {
    969                                throw new IllegalArgumentException("Need two datasets for complex dataset type");
    970                        }
    971                        c = new ComplexFloatDataset(a[0], a[1]);
    972                } else if (ComplexDoubleDataset.class.equals(clazz)) {
    973                        if (a.length != 2) {
    974                                throw new IllegalArgumentException("Need two datasets for complex dataset type");
    975                        }
    976                        c = new ComplexDoubleDataset(a[0], a[1]);
    977                } else {
    978                        utilsLogger.error("Dataset of unsupported type!");
    979                }
    980
    981                return c;
    982        }
    983
    984        /**
    985         * Cast array of datasets to a compound dataset
    986         *
    987         * @param a
    988         *            The datasets to be cast.
    989         * @return compound dataset of given type
    990         * @deprecated Please use the class-based methods in DatasetUtils,
    991         *             such as {@link #cast(Class, Dataset...)}
    992         */
    993        @Deprecated
    994        public static CompoundDataset cast(final Dataset[] a, final int dtype) {
    995                return cast(DTypeUtils.getInterface(dtype), a);
    996        }
    997
    998        /**
    999         * Make a dataset unsigned by promoting it to a wider dataset type and unwrapping the signs
    1000         * of its contents
    1001         * @param a
    1002         * @return unsigned dataset or original if it is not an integer dataset
    1003         */
    1004        public static Dataset makeUnsigned(IDataset a) {
    1005                return makeUnsigned(a, false);
    1006        }
    1007
    1008        /**
    1009         * Make a dataset unsigned by promoting it to a wider dataset type and unwrapping the signs
    1010         * of its contents
    1011         * @param a
    1012         * @param check if true, then check for negative values
    1013         * @return unsigned dataset or original if it is not an integer dataset or it has been check for negative numbers
    1014         * @since 2.1
    1015         */
    1016        public static Dataset makeUnsigned(IDataset a, boolean check) {
    1017                Dataset d = convertToDataset(a);
    1018
    1019                if (d.hasFloatingPointElements()) {
    1020                        return d;
    1021                }
    1022                if (check && d.min(true).longValue() >= 0) {
    1023                        return d;
    1024                }
    1025
    1026                int dtype = d.getDType();
    1027                switch (dtype) {
    1028                case Dataset.INT32:
    1029                        d = new LongDataset(d);
    1030                        unwrapUnsigned(d, 32);
    1031                        break;
    1032                case Dataset.INT16:
    1033                        d = new IntegerDataset(d);
    1034                        unwrapUnsigned(d, 16);
    1035                        break;
    1036                case Dataset.INT8:
    1037                        d = new ShortDataset(d);
    1038                        unwrapUnsigned(d, 8);
    1039                        break;
    1040                case Dataset.ARRAYINT32:
    1041                        d = new CompoundLongDataset(d);
    1042                        unwrapUnsigned(d, 32);
    1043                        break;
    1044                case Dataset.ARRAYINT16:
    1045                        d = new CompoundIntegerDataset(d);
    1046                        unwrapUnsigned(d, 16);
    1047                        break;
    1048                case Dataset.ARRAYINT8:
    1049                        d = new CompoundShortDataset(d);
    1050                        unwrapUnsigned(d, 8);
    1051                        break;
    1052                }
    1053                return d;
    1054        }
    1055
    1056        /**
    1057         * Unwrap dataset elements so that all elements are unsigned
    1058         * @param a dataset
    1059         * @param bitWidth width of original primitive in bits
    1060         */
    1061        public static void unwrapUnsigned(Dataset a, final int bitWidth) {
    1062                final int dtype = a.getDType();
    1063                final double dv = 1L << bitWidth;
    1064                final int isize = a.getElementsPerItem();
    1065                IndexIterator it = a.getIterator();
    1066
    1067                switch (dtype) {
    1068                case Dataset.BOOL:
    1069                        break;
    1070                case Dataset.INT8:
    1071                        break;
    1072                case Dataset.INT16:
    1073                        ShortDataset sds = (ShortDataset) a;
    1074                        final short soffset = (short) dv;
    1075                        while (it.hasNext()) {
    1076                                final short x = sds.getAbs(it.index);
    1077                                if (x < 0)
    1078                                        sds.setAbs(it.index, (short) (x + soffset));
    1079                        }
    1080                        break;
    1081                case Dataset.INT32:
    1082                        IntegerDataset ids = (IntegerDataset) a;
    1083                        final int ioffset = (int) dv;
    1084                        while (it.hasNext()) {
    1085                                final int x = ids.getAbs(it.index);
    1086                                if (x < 0)
    1087                                        ids.setAbs(it.index, x + ioffset);
    1088                        }
    1089                        break;
    1090                case Dataset.INT64:
    1091                        LongDataset lds = (LongDataset) a;
    1092                        final long loffset = (long) dv;
    1093                        while (it.hasNext()) {
    1094                                final long x = lds.getAbs(it.index);
    1095                                if (x < 0)
    1096                                        lds.setAbs(it.index, x + loffset);
    1097                        }
    1098                        break;
    1099                case Dataset.FLOAT32:
    1100                        FloatDataset fds = (FloatDataset) a;
    1101                        final float foffset = (float) dv;
    1102                        while (it.hasNext()) {
    1103                                final float x = fds.getAbs(it.index);
    1104                                if (x < 0)
    1105                                        fds.setAbs(it.index, x + foffset);
    1106                        }
    1107                        break;
    1108                case Dataset.FLOAT64:
    1109                        DoubleDataset dds = (DoubleDataset) a;
    1110                        final double doffset = dv;
    1111                        while (it.hasNext()) {
    1112                                final double x = dds.getAbs(it.index);
    1113                                if (x < 0)
    1114                                        dds.setAbs(it.index, x + doffset);
    1115                        }
    1116                        break;
    1117                case Dataset.ARRAYINT8:
    1118                        break;
    1119                case Dataset.ARRAYINT16:
    1120                        CompoundShortDataset csds = (CompoundShortDataset) a;
    1121                        final short csoffset = (short) dv;
    1122                        final short[] csa = new short[isize];
    1123                        while (it.hasNext()) {
    1124                                csds.getAbs(it.index, csa);
    1125                                boolean dirty = false;
    1126                                for (int i = 0; i < isize; i++) {
    1127                                        short x = csa[i];
    1128                                        if (x < 0) {
    1129                                                csa[i] = (short) (x + csoffset);
    1130                                                dirty = true;
    1131                                        }
    1132                                }
    1133                                if (dirty)
    1134                                        csds.setAbs(it.index, csa);
    1135                        }
    1136                        break;
    1137                case Dataset.ARRAYINT32:
    1138                        CompoundIntegerDataset cids = (CompoundIntegerDataset) a;
    1139                        final int cioffset = (int) dv;
    1140                        final int[] cia = new int[isize];
    1141                        while (it.hasNext()) {
    1142                                cids.getAbs(it.index, cia);
    1143                                boolean dirty = false;
    1144                                for (int i = 0; i < isize; i++) {
    1145                                        int x = cia[i];
    1146                                        if (x < 0) {
    1147                                                cia[i] = x + cioffset;
    1148                                                dirty = true;
    1149                                        }
    1150                                }
    1151                                if (dirty)
    1152                                        cids.setAbs(it.index, cia);
    1153                        }
    1154                        break;
    1155                case Dataset.ARRAYINT64:
    1156                        CompoundLongDataset clds = (CompoundLongDataset) a;
    1157                        final long cloffset = (long) dv;
    1158                        final long[] cla = new long[isize];
    1159                        while (it.hasNext()) {
    1160                                clds.getAbs(it.index, cla);
    1161                                boolean dirty = false;
    1162                                for (int i = 0; i < isize; i++) {
    1163                                        long x = cla[i];
    1164                                        if (x < 0) {
    1165                                                cla[i] = x + cloffset;
    1166                                                dirty = true;
    1167                                        }
    1168                                }
    1169                                if (dirty)
    1170                                        clds.setAbs(it.index, cla);
    1171                        }
    1172                        break;
    1173                default:
    1174                        utilsLogger.error("Dataset of unsupported type for this method");
    1175                        break;
    1176                }
    1177        }
    1178
    1179        /**
    1180         * @param clazz dataset class
    1181         * @param rows
    1182         * @param cols
    1183         * @param offset
    1184         * @param dtype
    1185         * @return a new 2d dataset of given shape and class, filled with ones on the (offset) diagonal
    1186         * @since 2.3
    1187         */
    1188        public static <T extends Dataset> T eye(final Class<T> clazz, final int rows, final int cols, final int offset) {
    1189                int[] shape = new int[] {rows, cols};
    1190                T a = DatasetFactory.zeros(clazz, shape);
    1191
    1192                int[] pos = new int[] {0, offset};
    1193                while (pos[1] < 0) {
    1194                        pos[0]++;
    1195                        pos[1]++;
    1196                }
    1197                while (pos[0] < rows && pos[1] < cols) {
    1198                        a.set(1, pos);
    1199                        pos[0]++;
    1200                        pos[1]++;
    1201                }
    1202
    1203                return a;
    1204        }
    1205
    1206        /**
    1207         * @param rows
    1208         * @param cols
    1209         * @param offset
    1210         * @param dtype
    1211         * @return a new 2d dataset of given shape and type, filled with ones on the (offset) diagonal
    1212         * @deprecated Please use the class-based methods in DatasetUtils,
    1213         *             such as {@link #eye(Class, int, int, int)}
    1214         */
    1215        @Deprecated
    1216        public static Dataset eye(final int rows, final int cols, final int offset, final int dtype) {
    1217                int[] shape = new int[] {rows, cols};
    1218                Dataset a = DatasetFactory.zeros(shape, dtype);
    1219
    1220                int[] pos = new int[] {0, offset};
    1221                while (pos[1] < 0) {
    1222                        pos[0]++;
    1223                        pos[1]++;
    1224                }
    1225                while (pos[0] < rows && pos[1] < cols) {
    1226                        a.set(1, pos);
    1227                        pos[0]++;
    1228                        pos[1]++;
    1229                }
    1230
    1231                return a;
    1232        }
    1233
    1234        /**
    1235         * Create a (off-)diagonal matrix from items in dataset
    1236         * @param a
    1237         * @param offset
    1238         * @return diagonal matrix
    1239         */
    1240        public static <T extends Dataset> T diag(final T a, final int offset) {
    1241                final int rank = a.getRank();
    1242                final int is = a.getElementsPerItem();
    1243
    1244                if (rank == 0 || rank > 2) {
    1245                        utilsLogger.error("Rank of dataset should be one or two");
    1246                        throw new IllegalArgumentException("Rank of dataset should be one or two");
    1247                }
    1248
    1249                Dataset result;
    1250                final int[] shape = a.getShapeRef();
    1251                if (rank == 1) {
    1252                        int side = shape[0] + Math.abs(offset);
    1253                        int[] pos = new int[] {side, side};
    1254                        result = DatasetFactory.zeros(is, a.getClass(), pos);
    1255                        if (offset >= 0) {
    1256                                pos[0] = 0;
    1257                                pos[1] = offset;
    1258                        } else {
    1259                                pos[0] = -offset;
    1260                                pos[1] = 0;
    1261                        }
    1262                        int i = 0;
    1263                        while (pos[0] < side && pos[1] < side) {
    1264                                result.set(a.getObject(i++), pos);
    1265                                pos[0]++;
    1266                                pos[1]++;
    1267                        }
    1268                } else {
    1269                        int side = offset >= 0 ? Math.min(shape[0], shape[1]-offset) : Math.min(shape[0]+offset, shape[1]);
    1270                        if (side < 0)
    1271                                side = 0;
    1272                        result = DatasetFactory.zeros(is, a.getClass(), side);
    1273
    1274                        if (side > 0) {
    1275                                int[] pos = offset >= 0 ? new int[] { 0, offset } : new int[] { -offset, 0 };
    1276                                int i = 0;
    1277                                while (pos[0] < shape[0] && pos[1] < shape[1]) {
    1278                                        result.set(a.getObject(pos), i++);
    1279                                        pos[0]++;
    1280                                        pos[1]++;
    1281                                }
    1282                        }
    1283                }
    1284
    1285                return (T) result;
    1286        }
    1287
    1288        /**
    1289         * Slice (or fully load), if necessary, a lazy dataset, otherwise take a slice view and
    1290         * convert to our dataset implementation. If a slice is necessary, this may cause resource
    1291         * problems when used on large datasets and throw runtime exceptions
    1292         * @param lazy can be null
    1293         * @return Converted dataset or null
    1294         * @throws DatasetException
    1295         */
    1296        public static Dataset sliceAndConvertLazyDataset(ILazyDataset lazy) throws DatasetException {
    1297                if (lazy == null)
    1298                        return null;
    1299
    1300                IDataset data = lazy instanceof IDataset ? (IDataset) lazy.getSliceView() : lazy.getSlice();
    1301
    1302                return convertToDataset(data);
    1303        }
    1304
    1305        /**
    1306         * Convert (if necessary) a dataset obeying the interface to our implementation
    1307         * @param data can be null
    1308         * @return Converted dataset or null
    1309         */
    1310        public static Dataset convertToDataset(IDataset data) {
    1311                if (data == null)
    1312                        return null;
    1313
    1314                if (data instanceof Dataset) {
    1315                        return (Dataset) data;
    1316                }
    1317
    1318                final int isize = data.getElementsPerItem();
    1319                Class<? extends Dataset> clazz = InterfaceUtils.getInterfaceFromClass(isize, data.getElementClass());
    1320                if (isize <= 0) {
    1321                        throw new IllegalArgumentException("Datasets with " + isize + " elements per item not supported");
    1322                }
    1323
    1324                final Dataset result = DatasetFactory.zeros(isize, clazz, data.getShape());
    1325                result.setName(data.getName());
    1326
    1327                final IndexIterator it = result.getIterator(true);
    1328                final int[] pos = it.getPos();
    1329                switch (DTypeUtils.getDType(clazz)) {
    1330                case Dataset.BOOL:
    1331                        while (it.hasNext()) {
    1332                                result.setObjectAbs(it.index, data.getBoolean(pos));
    1333                        }
    1334                        break;
    1335                case Dataset.INT8:
    1336                        while (it.hasNext()) {
    1337                                result.setObjectAbs(it.index, data.getByte(pos));
    1338                        }
    1339                        break;
    1340                case Dataset.INT16:
    1341                        while (it.hasNext()) {
    1342                                result.setObjectAbs(it.index, data.getShort(pos));
    1343                        }
    1344                        break;
    1345                case Dataset.INT32:
    1346                        while (it.hasNext()) {
    1347                                result.setObjectAbs(it.index, data.getInt(pos));
    1348                        }
    1349                        break;
    1350                case Dataset.INT64:
    1351                        while (it.hasNext()) {
    1352                                result.setObjectAbs(it.index, data.getLong(pos));
    1353                        }
    1354                        break;
    1355                case Dataset.FLOAT32:
    1356                        while (it.hasNext()) {
    1357                                result.setObjectAbs(it.index, data.getFloat(pos));
    1358                        }
    1359                        break;
    1360                case Dataset.FLOAT64:
    1361                        while (it.hasNext()) {
    1362                                result.setObjectAbs(it.index, data.getDouble(pos));
    1363                        }
    1364                        break;
    1365                default:
    1366                        while (it.hasNext()) {
    1367                                result.setObjectAbs(it.index, data.getObject(pos));
    1368                        }
    1369                        break;
    1370                }
    1371
    1372                result.setErrors(data.getErrors());
    1373                return result;
    1374        }
    1375
    1376        /**
    1377         * Create a compound dataset from given datasets
    1378         * @param datasets
    1379         * @return compound dataset or null if none given
    1380         */
    1381        public static CompoundDataset createCompoundDataset(final Dataset... datasets) {
    1382                if (datasets == null || datasets.length == 0)
    1383                        return null;
    1384
    1385                return createCompoundDataset(datasets[0].getDType(), datasets);
    1386        }
    1387
    1388        /**
    1389         * Create a compound dataset from given datasets
    1390         * @param clazz dataset class
    1391         * @param datasets
    1392         * @return compound dataset or null if none given
    1393         */
    1394        public static <T extends CompoundDataset> T createCompoundDataset(Class<T> clazz, final Dataset... datasets) {
    1395                return (T) createCompoundDataset(DTypeUtils.getDType(clazz), datasets);
    1396        }
    1397
    1398        /**
    1399         * Create a compound dataset from given datasets
    1400         * @param dtype
    1401         * @param datasets
    1402         * @return compound dataset or null if none given
    1403         * @deprecated Please use the class-based methods in DatasetUtils,
    1404         *             such as {@link #createCompoundDataset(Class, Dataset...)}
    1405         */
    1406        @Deprecated
    1407        public static CompoundDataset createCompoundDataset(final int dtype, final Dataset... datasets) {
    1408                if (datasets == null || datasets.length == 0)
    1409                        return null;
    1410
    1411                switch (dtype) {
    1412                case Dataset.INT8:
    1413                case Dataset.ARRAYINT8:
    1414                        return new CompoundByteDataset(datasets);
    1415                case Dataset.INT16:
    1416                case Dataset.ARRAYINT16:
    1417                        return new CompoundShortDataset(datasets);
    1418                case Dataset.INT32:
    1419                case Dataset.ARRAYINT32:
    1420                        return new CompoundIntegerDataset(datasets);
    1421                case Dataset.INT64:
    1422                case Dataset.ARRAYINT64:
    1423                        return new CompoundLongDataset(datasets);
    1424                case Dataset.FLOAT32:
    1425                case Dataset.ARRAYFLOAT32:
    1426                        return new CompoundFloatDataset(datasets);
    1427                case Dataset.FLOAT64:
    1428                case Dataset.ARRAYFLOAT64:
    1429                        return new CompoundDoubleDataset(datasets);
    1430                case Dataset.COMPLEX64:
    1431                case Dataset.COMPLEX128:
    1432                        if (datasets.length > 2) {
    1433                                utilsLogger.error("At most two datasets are allowed");
    1434                                throw new IllegalArgumentException("At most two datasets are allowed");
    1435                        } else if (datasets.length == 2) {
    1436                                return dtype == Dataset.COMPLEX64 ? new ComplexFloatDataset(datasets[0], datasets[1]) : new ComplexDoubleDataset(datasets[0], datasets[1]);
    1437                        }
    1438                        return dtype == Dataset.COMPLEX64 ? new ComplexFloatDataset(datasets[0]) : new ComplexDoubleDataset(datasets[0]);
    1439                case Dataset.RGB:
    1440                        if (datasets.length == 1) {
    1441                                return new RGBDataset(datasets[0]);
    1442                        } else if (datasets.length == 3) {
    1443                                return new RGBDataset(datasets[0], datasets[1], datasets[2]);
    1444                        } else {
    1445                                utilsLogger.error("Only one or three datasets are allowed to create a RGB dataset");
    1446                                throw new IllegalArgumentException("Only one or three datasets are allowed to create a RGB dataset");
    1447                        }
    1448                default:
    1449                        utilsLogger.error("Dataset type not supported for this operation");
    1450                        throw new UnsupportedOperationException("Dataset type not supported");
    1451                }
    1452        }
    1453
    1454        /**
    1455         * Create a compound dataset from given dataset
    1456         * @param dataset
    1457         * @param itemSize
    1458         * @return compound dataset
    1459         */
    1460        public static CompoundDataset createCompoundDataset(final Dataset dataset, final int itemSize) {
    1461                int[] shape = dataset.getShapeRef();
    1462                int[] nshape = shape;
    1463                if (shape != null && itemSize > 1) {
    1464                        int size = ShapeUtils.calcSize(shape);
    1465                        if (size % itemSize != 0) {
    1466                                throw new IllegalArgumentException("Input dataset has number of items that is not a multiple of itemSize");
    1467                        }
    1468                        int d = shape.length;
    1469                        int l = 1;
    1470                        while (--d >= 0) {
    1471                                l *= shape[d];
    1472                                if (l % itemSize == 0) {
    1473                                        break;
    1474                                }
    1475                        }
    1476                        assert d >= 0;
    1477                        nshape = new int[d + 1];
    1478                        for (int i = 0; i < d; i++) {
    1479                                nshape[i] = shape[i];
    1480                        }
    1481                        nshape[d] = l / itemSize;
    1482                }
    1483                switch (dataset.getDType()) {
    1484                case Dataset.INT8:
    1485                        return new CompoundByteDataset(itemSize, (byte[]) dataset.getBuffer(), nshape);
    1486                case Dataset.INT16:
    1487                        return new CompoundShortDataset(itemSize, (short[]) dataset.getBuffer(), nshape);
    1488                case Dataset.INT32:
    1489                        return new CompoundIntegerDataset(itemSize, (int[]) dataset.getBuffer(), nshape);
    1490                case Dataset.INT64:
    1491                        return new CompoundLongDataset(itemSize, (long[]) dataset.getBuffer(), nshape);
    1492                case Dataset.FLOAT32:
    1493                        return new CompoundFloatDataset(itemSize, (float[]) dataset.getBuffer(), nshape);
    1494                case Dataset.FLOAT64:
    1495                        return new CompoundDoubleDataset(itemSize, (double[]) dataset.getBuffer(), nshape);
    1496                default:
    1497                        utilsLogger.error("Dataset type not supported for this operation");
    1498                        throw new UnsupportedOperationException("Dataset type not supported");
    1499                }
    1500        }
    1501
    1502
    1503        /**
    1504         * Create a compound dataset by using last axis as elements of an item
    1505         * @param a
    1506         * @param shareData if true, then share data
    1507         * @return compound dataset
    1508         */
    1509        public static CompoundDataset createCompoundDatasetFromLastAxis(final Dataset a, final boolean shareData) {
    1510                switch (a.getDType()) {
    1511                case Dataset.INT8:
    1512                        return CompoundByteDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1513                case Dataset.INT16:
    1514                        return CompoundShortDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1515                case Dataset.INT32:
    1516                        return CompoundIntegerDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1517                case Dataset.INT64:
    1518                        return CompoundLongDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1519                case Dataset.FLOAT32:
    1520                        return CompoundFloatDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1521                case Dataset.FLOAT64:
    1522                        return CompoundDoubleDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1523                default:
    1524                        utilsLogger.error("Dataset type not supported for this operation");
    1525                        throw new UnsupportedOperationException("Dataset type not supported");
    1526                }
    1527        }
    1528
    1529        /**
    1530         * Create a dataset from a compound dataset by using elements of an item as last axis
    1531         * <p>
    1532         * In the case where the number of elements is one, the last axis is squeezed out.
    1533         * @param a
    1534         * @param shareData if true, then share data
    1535         * @return non-compound dataset
    1536         */
    1537        public static Dataset createDatasetFromCompoundDataset(final CompoundDataset a, final boolean shareData) {
    1538                return a.asNonCompoundDataset(shareData);
    1539        }
    1540
    1541        /**
    1542         * Create a copy that has been coerced to an appropriate dataset type
    1543         * depending on the input object's class
    1544         *
    1545         * @param a
    1546         * @param obj
    1547         * @return coerced copy of dataset
    1548         */
    1549        public static Dataset coerce(Dataset a, Object obj) {
    1550                return cast(InterfaceUtils.getBestInterface(a.getClass(), InterfaceUtils.getInterface(obj)), a.clone());
    1551        }
    1552
    1553        /**
    1554         * Function that returns a normalised dataset which is bounded between 0 and 1
    1555         * @param a dataset
    1556         * @return normalised dataset
    1557         */
    1558        public static Dataset norm(Dataset a) {
    1559                double amin = a.min().doubleValue();
    1560                double aptp = a.max().doubleValue() - amin;
    1561                Dataset temp = Maths.subtract(a, amin);
    1562                temp.idivide(aptp);
    1563                return temp;
    1564        }
    1565
    1566        /**
    1567         * Function that returns a normalised compound dataset which is bounded between 0 and 1. There
    1568         * are (at least) two ways to normalise a compound dataset: per element - extrema for each element
    1569         * in a compound item is used, i.e. many min/max pairs; over all elements - extrema for all elements
    1570         * is used, i.e. one min/max pair.
    1571         * @param a dataset
    1572         * @param overAllElements if true, then normalise over all elements in each item
    1573         * @return normalised dataset
    1574         */
    1575        public static CompoundDataset norm(CompoundDataset a, boolean overAllElements) {
    1576                double[] amin = a.minItem();
    1577                double[] amax = a.maxItem();
    1578                final int is = a.getElementsPerItem();
    1579                Dataset result;
    1580
    1581                if (overAllElements) {
    1582                        Arrays.sort(amin);
    1583                        Arrays.sort(amax);
    1584                        double aptp = amax[0] - amin[0];
    1585
    1586                        result = Maths.subtract(a, amin[0]);
    1587                        result.idivide(aptp);
    1588                } else {
    1589                        double[] aptp = new double[is];
    1590                        for (int j = 0; j < is; j++) {
    1591                                aptp[j] = amax[j] - amin[j];
    1592                        }
    1593
    1594                        result = Maths.subtract(a, amin);
    1595                        result.idivide(aptp);
    1596                }
    1597                return (CompoundDataset) result;
    1598        }
    1599
    1600        /**
    1601         * Function that returns a normalised dataset which is bounded between 0 and 1
    1602         * and has been distributed on a log10 scale
    1603         * @param a dataset
    1604         * @return normalised dataset
    1605         */
    1606        public static Dataset lognorm(Dataset a) {
    1607                double amin = a.min().doubleValue();
    1608                double aptp = Math.log10(a.max().doubleValue() - amin + 1.);
    1609                Dataset temp = Maths.subtract(a, amin - 1.);
    1610                temp = Maths.log10(temp);
    1611                temp = Maths.divide(temp, aptp);
    1612                return temp;
    1613        }
    1614
    1615        /**
    1616         * Function that returns a normalised dataset which is bounded between 0 and 1
    1617         * and has been distributed on a natural log scale
    1618         * @param a dataset
    1619         * @return normalised dataset
    1620         */
    1621        public static Dataset lnnorm(Dataset a) {
    1622                double amin = a.min().doubleValue();
    1623                double aptp = Math.log(a.max().doubleValue() - amin + 1.);
    1624                Dataset temp = Maths.subtract(a, amin - 1.);
    1625                temp = Maths.log(temp);
    1626                temp = Maths.divide(temp, aptp);
    1627                return temp;
    1628        }
    1629
    1630        /**
    1631         * Construct a list of datasets where each represents a coordinate varying over the hypergrid
    1632         * formed by the input list of axes
    1633         *
    1634         * @param axes an array of 1D datasets representing axes
    1635         * @return a list of coordinate datasets
    1636         */
    1637        public static List<Dataset> meshGrid(final Dataset... axes) {
    1638                List<Dataset> result = new ArrayList<Dataset>();
    1639                int rank = axes.length;
    1640
    1641                if (rank < 2) {
    1642                        utilsLogger.error("Two or more axes datasets are required");
    1643                        throw new IllegalArgumentException("Two or more axes datasets are required");
    1644                }
    1645
    1646                int[] nshape = new int[rank];
    1647
    1648                for (int i = 0; i < rank; i++) {
    1649                        Dataset axis = axes[i];
    1650                        if (axis.getRank() != 1) {
    1651                                utilsLogger.error("Given axis is not 1D");
    1652                                throw new IllegalArgumentException("Given axis is not 1D");
    1653                        }
    1654                        nshape[i] = axis.getSize();
    1655                }
    1656
    1657                for (int i = 0; i < rank; i++) {
    1658                        Dataset axis = axes[i];
    1659                        Dataset coord = DatasetFactory.zeros(axis.getClass(), nshape);
    1660                        result.add(coord);
    1661
    1662                        final int alen = axis.getSize();
    1663                        for (int j = 0; j < alen; j++) {
    1664                                final Object obj = axis.getObjectAbs(j);
    1665                                PositionIterator pi = coord.getPositionIterator(i);
    1666                                final int[] pos = pi.getPos();
    1667
    1668                                pos[i] = j;
    1669                                while (pi.hasNext()) {
    1670                                        coord.set(obj, pos);
    1671                                }
    1672                        }
    1673                }
    1674
    1675                return result;
    1676        }
    1677
    1678        /**
    1679         * Generate an index dataset for given dataset where sub-datasets contain index values
    1680         *
    1681         * @return an index dataset
    1682         */
    1683        public static IntegerDataset indices(int... shape) {
    1684                // now create another dataset to plot against
    1685                final int rank = shape.length;
    1686                int[] nshape = new int[rank+1];
    1687                nshape[0] = rank;
    1688                for (int i = 0; i < rank; i++) {
    1689                        nshape[i+1] = shape[i];
    1690                }
    1691
    1692                IntegerDataset index = new IntegerDataset(nshape);
    1693
    1694                if (rank == 1) {
    1695                        final int alen = shape[0];
    1696                        int[] pos = new int[2];
    1697                        for (int j = 0; j < alen; j++) {
    1698                                pos[1] = j;
    1699                                index.set(j, pos);
    1700                        }
    1701                } else {
    1702                        for (int i = 1; i <= rank; i++) {
    1703                                final int alen = nshape[i];
    1704                                for (int j = 0; j < alen; j++) {
    1705                                        PositionIterator pi = index.getPositionIterator(0, i);
    1706                                        final int[] pos = pi.getPos();
    1707
    1708                                        pos[0] = i-1;
    1709                                        pos[i] = j;
    1710                                        while (pi.hasNext()) {
    1711                                                index.set(j, pos);
    1712                                        }
    1713                                }
    1714                        }
    1715                }
    1716                return index;
    1717        }
    1718
    1719        /**
    1720         * Get the centroid value of a dataset, this function works out the centroid in every direction
    1721         *
    1722         * @param a
    1723         *            the dataset to be analysed
    1724         * @param bases the optional array of base coordinates to use as weights.
    1725         * This defaults to the mid-point of indices
    1726         * @return a double array containing the centroid for each dimension
    1727         */
    1728        public static double[] centroid(Dataset a, Dataset... bases) {
    1729                int rank = a.getRank();
    1730                if (bases.length > 0 && bases.length != rank) {
    1731                        throw new IllegalArgumentException("Number of bases must be zero or match rank of dataset");
    1732                }
    1733
    1734                int[] shape = a.getShapeRef();
    1735                if (bases.length == rank) {
    1736                        for (int i = 0; i < rank; i++) {
    1737                                Dataset b = bases[i];
    1738                                if (b.getRank() != 1 && b.getSize() != shape[i]) {
    1739                                        throw new IllegalArgumentException("A base does not have shape to match given dataset");
    1740                                }
    1741                        }
    1742                }
    1743
    1744                double[] dc = new double[rank];
    1745                if (rank == 0)
    1746                        return dc;
    1747
    1748                final PositionIterator iter = new PositionIterator(shape);
    1749                final int[] pos = iter.getPos();
    1750
    1751                double tsum = 0.0;
    1752                while (iter.hasNext()) {
    1753                        double val = a.getDouble(pos);
    1754                        tsum += val;
    1755                        for (int d = 0; d < rank; d++) {
    1756                                Dataset b = bases.length == 0 ? null : bases[d];
    1757                                if (b == null) {
    1758                                        dc[d] += (pos[d] + 0.5) * val;
    1759                                } else {
    1760                                        dc[d] += b.getElementDoubleAbs(pos[d]) * val;
    1761                                }
    1762                        }
    1763                }
    1764
    1765                for (int d = 0; d < rank; d++) {
    1766                        dc[d] /= tsum;
    1767                }
    1768                return dc;
    1769        }
    1770
    1771        /**
    1772         * Find linearly-interpolated crossing points where the given dataset crosses the given value
    1773         *
    1774         * @param d
    1775         * @param value
    1776         * @return list of interpolated indices
    1777         */
    1778        public static List<Double> crossings(Dataset d, double value) {
    1779                if (d.getRank() != 1) {
    1780                        utilsLogger.error("Only 1d datasets supported");
    1781                        throw new UnsupportedOperationException("Only 1d datasets supported");
    1782                }
    1783                List<Double> results = new ArrayList<Double>();
    1784
    1785                // run through all pairs of points on the line and see if value lies within
    1786                IndexIterator it = d.getIterator();
    1787                double y1, y2;
    1788
    1789                y2 = it.hasNext() ? d.getElementDoubleAbs(it.index) : 0;
    1790                double x = 1;
    1791                while (it.hasNext()) {
    1792                        y1 = y2;
    1793                        y2 = d.getElementDoubleAbs(it.index);
    1794                        // check if value lies within pair [y1, y2]
    1795                        if ((y1 <= value && value < y2) || (y1 > value && y2 <= value)) {
    1796                                final double f = (value - y2)/(y2 - y1); // negative distance from right to left
    1797                                results.add(x + f);
    1798                        }
    1799                        x++;
    1800                }
    1801                if (y2 == value) { // add end point of it intersects
    1802                        results.add(x);
    1803                }
    1804
    1805                return results;
    1806        }
    1807
    1808        /**
    1809         * Find x values of all the crossing points of the dataset with the given y value
    1810         *
    1811         * @param xAxis
    1812         *            Dataset of the X axis that needs to be looked at
    1813         * @param yAxis
    1814         *            Dataset of the Y axis that needs to be looked at
    1815         * @param yValue
    1816         *            The y value the X values are required for
    1817         * @return An list of doubles containing all the X coordinates of where the line crosses
    1818         */
    1819        public static List<Double> crossings(Dataset xAxis, Dataset yAxis, double yValue) {
    1820                if (xAxis.getSize() > yAxis.getSize()) {
    1821                        throw new IllegalArgumentException(
    1822                                        "Number of values of yAxis must as least be equal to the number of values of xAxis");
    1823                }
    1824
    1825                List<Double> results = new ArrayList<Double>();
    1826
    1827                List<Double> indices = crossings(yAxis, yValue);
    1828
    1829                for (double xi : indices) {
    1830                        results.add(Maths.interpolate(xAxis, xi));
    1831                }
    1832                return results;
    1833        }
    1834
    1835        /**
    1836         * Function that uses the crossings function but prunes the result, so that multiple crossings within a
    1837         * certain proportion of the overall range of the x values
    1838         *
    1839         * @param xAxis
    1840         *            Dataset of the X axis
    1841         * @param yAxis
    1842         *            Dataset of the Y axis
    1843         * @param yValue
    1844         *            The y value the x values are required for
    1845         * @param xRangeProportion
    1846         *            The proportion of the overall x spread used to prune result
    1847         * @return A list containing all the unique crossing points
    1848         */
    1849        public static List<Double> crossings(Dataset xAxis, Dataset yAxis, double yValue, double xRangeProportion) {
    1850                // get the values found
    1851                List<Double> vals = crossings(xAxis, yAxis, yValue);
    1852
    1853                // use the proportion to calculate the error spacing
    1854                double error = xRangeProportion * xAxis.peakToPeak().doubleValue();
    1855
    1856                int i = 0;
    1857                // now go through and check for groups of three crossings which are all
    1858                // within the boundaries
    1859                while (i <= vals.size() - 3) {
    1860                        double v1 = Math.abs(vals.get(i) - vals.get(i + 2));
    1861                        if (v1 < error) {
    1862                                // these 3 points should be treated as one
    1863                                // make the first point equal to the average of them all
    1864                                vals.set(i + 2, ((vals.get(i) + vals.get(i + 1) + vals.get(i + 2)) / 3.0));
    1865                                // remove the other offending points
    1866                                vals.remove(i);
    1867                                vals.remove(i);
    1868                        } else {
    1869                                i++;
    1870                        }
    1871                }
    1872
    1873                // once the thinning process has been completed, return the pruned list
    1874                return vals;
    1875        }
    1876
    1877        // recursive function
    1878        private static void setRow(Object row, Dataset a, int... pos) {
    1879                final int l = Array.getLength(row);
    1880                final int rank = pos.length;
    1881                final int[] npos = Arrays.copyOf(pos, rank+1);
    1882                Object r;
    1883                if (rank+1 < a.getRank()) {
    1884                        for (int i = 0; i < l; i++) {
    1885                                npos[rank] = i;
    1886                                r = Array.get(row, i);
    1887                                setRow(r, a, npos);
    1888                        }
    1889                } else {
    1890                        for (int i = 0; i < l; i++) {
    1891                                npos[rank] = i;
    1892                                r = a.getObject(npos);
    1893                                Array.set(row, i, r);
    1894                        }
    1895                }
    1896        }
    1897
    1898        /**
    1899         * Create Java array (of arrays) from dataset
    1900         * @param a dataset
    1901         * @return Java array (of arrays...)
    1902         */
    1903        public static Object createJavaArray(Dataset a) {
    1904                if (a.getElementsPerItem() > 1) {
    1905                        a = createDatasetFromCompoundDataset((CompoundDataset) a, true);
    1906                }
    1907                Object matrix;
    1908
    1909                int[] shape = a.getShapeRef();
    1910                switch (a.getDType()) {
    1911                case Dataset.BOOL:
    1912                        matrix = Array.newInstance(boolean.class, shape);
    1913                        break;
    1914                case Dataset.INT8:
    1915                        matrix = Array.newInstance(byte.class, shape);
    1916                        break;
    1917                case Dataset.INT16:
    1918                        matrix = Array.newInstance(short.class, shape);
    1919                        break;
    1920                case Dataset.INT32:
    1921                        matrix = Array.newInstance(int.class, shape);
    1922                        break;
    1923                case Dataset.INT64:
    1924                        matrix = Array.newInstance(long.class, shape);
    1925                        break;
    1926                case Dataset.FLOAT32:
    1927                        matrix = Array.newInstance(float.class, shape);
    1928                        break;
    1929                case Dataset.FLOAT64:
    1930                        matrix = Array.newInstance(double.class, shape);
    1931                        break;
    1932                default:
    1933                        utilsLogger.error("Dataset type not supported");
    1934                        throw new IllegalArgumentException("Dataset type not supported");
    1935                }
    1936
    1937                // populate matrix
    1938                setRow(matrix, a);
    1939                return matrix;
    1940        }
    1941
    1942        /**
    1943         * Removes NaNs and infinities from floating point datasets.
    1944         * All other dataset types are ignored.
    1945         *
    1946         * @param a dataset
    1947         * @param value replacement value
    1948         */
    1949        public static void removeNansAndInfinities(Dataset a, final Number value) {
    1950                if (a instanceof DoubleDataset) {
    1951                        final double dvalue = DTypeUtils.toReal(value);
    1952                        final DoubleDataset set = (DoubleDataset) a;
    1953                        final IndexIterator it = set.getIterator();
    1954                        final double[] data = set.getData();
    1955                        while (it.hasNext()) {
    1956                                double x = data[it.index];
    1957                                if (Double.isNaN(x) || Double.isInfinite(x))
    1958                                        data[it.index] = dvalue;
    1959                        }
    1960                } else if (a instanceof FloatDataset) {
    1961                        final float fvalue = (float) DTypeUtils.toReal(value);
    1962                        final FloatDataset set = (FloatDataset) a;
    1963                        final IndexIterator it = set.getIterator();
    1964                        final float[] data = set.getData();
    1965                        while (it.hasNext()) {
    1966                                float x = data[it.index];
    1967                                if (Float.isNaN(x) || Float.isInfinite(x))
    1968                                        data[it.index] = fvalue;
    1969                        }
    1970                } else if (a instanceof CompoundDoubleDataset) {
    1971                        final double dvalue = DTypeUtils.toReal(value);
    1972                        final CompoundDoubleDataset set = (CompoundDoubleDataset) a;
    1973                        final int is = set.getElementsPerItem();
    1974                        final IndexIterator it = set.getIterator();
    1975                        final double[] data = set.getData();
    1976                        while (it.hasNext()) {
    1977                                for (int j = 0; j < is; j++) {
    1978                                        double x = data[it.index + j];
    1979                                        if (Double.isNaN(x) || Double.isInfinite(x))
    1980                                                data[it.index + j] = dvalue;
    1981                                }
    1982                        }
    1983                } else if (a instanceof CompoundFloatDataset) {
    1984                        final float fvalue = (float) DTypeUtils.toReal(value);
    1985                        final CompoundFloatDataset set = (CompoundFloatDataset) a;
    1986                        final int is = set.getElementsPerItem();
    1987                        final IndexIterator it = set.getIterator();
    1988                        final float[] data = set.getData();
    1989                        while (it.hasNext()) {
    1990                                for (int j = 0; j < is; j++) {
    1991                                        float x = data[it.index + j];
    1992                                        if (Float.isNaN(x) || Float.isInfinite(x))
    1993                                                data[it.index + j] = fvalue;
    1994                                }
    1995                        }
    1996                }
    1997        }
    1998
    1999        /**
    2000         * Make floating point datasets contain only finite values. Infinities and NaNs are replaced
    2001         * by +/- MAX_VALUE and 0, respectively.
    2002         * All other dataset types are ignored.
    2003         *
    2004         * @param a dataset
    2005         */
    2006        public static void makeFinite(Dataset a) {
    2007                if (a instanceof DoubleDataset) {
    2008                        final DoubleDataset set = (DoubleDataset) a;
    2009                        final IndexIterator it = set.getIterator();
    2010                        final double[] data = set.getData();
    2011                        while (it.hasNext()) {
    2012                                final double x = data[it.index];
    2013                                if (Double.isNaN(x))
    2014                                        data[it.index] = 0;
    2015                                else if (Double.isInfinite(x))
    2016                                        data[it.index] = x > 0 ? Double.MAX_VALUE : -Double.MAX_VALUE;
    2017                        }
    2018                } else if (a instanceof FloatDataset) {
    2019                        final FloatDataset set = (FloatDataset) a;
    2020                        final IndexIterator it = set.getIterator();
    2021                        final float[] data = set.getData();
    2022                        while (it.hasNext()) {
    2023                                final float x = data[it.index];
    2024                                if (Float.isNaN(x))
    2025                                        data[it.index] = 0;
    2026                                else if (Float.isInfinite(x))
    2027                                        data[it.index] = x > 0 ? Float.MAX_VALUE : -Float.MAX_VALUE;
    2028                        }
    2029                } else if (a instanceof CompoundDoubleDataset) {
    2030                        final CompoundDoubleDataset set = (CompoundDoubleDataset) a;
    2031                        final int is = set.getElementsPerItem();
    2032                        final IndexIterator it = set.getIterator();
    2033                        final double[] data = set.getData();
    2034                        while (it.hasNext()) {
    2035                                for (int j = 0; j < is; j++) {
    2036                                        final double x = data[it.index + j];
    2037                                        if (Double.isNaN(x))
    2038                                                data[it.index + j] = 0;
    2039                                        else if (Double.isInfinite(x))
    2040                                                data[it.index + j] = x > 0 ? Double.MAX_VALUE : -Double.MAX_VALUE;
    2041                                }
    2042                        }
    2043                } else if (a instanceof CompoundFloatDataset) {
    2044                        final CompoundFloatDataset set = (CompoundFloatDataset) a;
    2045                        final int is = set.getElementsPerItem();
    2046                        final IndexIterator it = set.getIterator();
    2047                        final float[] data = set.getData();
    2048                        while (it.hasNext()) {
    2049                                for (int j = 0; j < is; j++) {
    2050                                        final float x = data[it.index + j];
    2051                                        if (Float.isNaN(x))
    2052                                                data[it.index + j] = 0;
    2053                                        else if (Float.isInfinite(x))
    2054                                                data[it.index + j] = x > 0 ? Float.MAX_VALUE : -Float.MAX_VALUE;
    2055                                }
    2056                        }
    2057                }
    2058        }
    2059
    2060        /**
    2061         * Find absolute index of first value in dataset that is equal to given number
    2062         * @param a
    2063         * @param n
    2064         * @return absolute index (if greater than a.getSize() then no value found)
    2065         */
    2066        public static int findIndexEqualTo(final Dataset a, final double n) {
    2067                IndexIterator iter = a.getIterator();
    2068                while (iter.hasNext()) {
    2069                        if (a.getElementDoubleAbs(iter.index) == n)
    2070                                break;
    2071                }
    2072
    2073                return iter.index;
    2074        }
    2075
    2076        /**
    2077         * Find absolute index of first value in dataset that is greater than given number
    2078         * @param a
    2079         * @param n
    2080         * @return absolute index (if greater than a.getSize() then no value found)
    2081         */
    2082        public static int findIndexGreaterThan(final Dataset a, final double n) {
    2083                IndexIterator iter = a.getIterator();
    2084                while (iter.hasNext()) {
    2085                        if (a.getElementDoubleAbs(iter.index) > n)
    2086                                break;
    2087                }
    2088
    2089                return iter.index;
    2090        }
    2091
    2092        /**
    2093         * Find absolute index of first value in dataset that is greater than or equal to given number
    2094         * @param a
    2095         * @param n
    2096         * @return absolute index (if greater than a.getSize() then no value found)
    2097         */
    2098        public static int findIndexGreaterThanOrEqualTo(final Dataset a, final double n) {
    2099                IndexIterator iter = a.getIterator();
    2100                while (iter.hasNext()) {
    2101                        if (a.getElementDoubleAbs(iter.index) >= n)
    2102                                break;
    2103                }
    2104
    2105                return iter.index;
    2106        }
    2107
    2108        /**
    2109         * Find absolute index of first value in dataset that is less than given number
    2110         * @param a
    2111         * @param n
    2112         * @return absolute index (if greater than a.getSize() then no value found)
    2113         */
    2114        public static int findIndexLessThan(final Dataset a, final double n) {
    2115                IndexIterator iter = a.getIterator();
    2116                while (iter.hasNext()) {
    2117                        if (a.getElementDoubleAbs(iter.index) < n)
    2118                                break;
    2119                }
    2120
    2121                return iter.index;
    2122        }
    2123
    2124        /**
    2125         * Find absolute index of first value in dataset that is less than or equal to given number
    2126         * @param a
    2127         * @param n
    2128         * @return absolute index (if greater than a.getSize() then no value found)
    2129         */
    2130        public static int findIndexLessThanOrEqualTo(final Dataset a, final double n) {
    2131                IndexIterator iter = a.getIterator();
    2132                while (iter.hasNext()) {
    2133                        if (a.getElementDoubleAbs(iter.index) <= n)
    2134                                break;
    2135                }
    2136
    2137                return iter.index;
    2138        }
    2139
    2140        /**
    2141         * Find first occurrences in one dataset of values given in another sorted dataset
    2142         * @param a
    2143         * @param values sorted 1D dataset of values to find
    2144         * @return absolute indexes of those first occurrences (-1 is used to indicate value not found)
    2145         */
    2146        public static IntegerDataset findFirstOccurrences(final Dataset a, final Dataset values) {
    2147                if (values.getRank() != 1) {
    2148                        throw new IllegalArgumentException("Values dataset must be 1D");
    2149                }
    2150                IntegerDataset indexes = new IntegerDataset(values.getSize());
    2151                indexes.fill(-1);
    2152
    2153                IndexIterator it = a.getIterator();
    2154                final int n = values.getSize();
    2155                if (values instanceof LongDataset) {
    2156                        while (it.hasNext()) {
    2157                                long x = a.getElementLongAbs(it.index);
    2158
    2159                                int l = 0; // binary search to find value in sorted dataset
    2160                                long vl = values.getLong(l);
    2161                                if (x <= vl) {
    2162                                        if (x == vl && indexes.getAbs(l) < 0)
    2163                                                indexes.setAbs(l, it.index);
    2164                                        continue;
    2165                                }
    2166                                int h = n - 1;
    2167                                long vh = values.getLong(h);
    2168                                if (x >= vh) {
    2169                                        if (x == vh && indexes.getAbs(h) < 0)
    2170                                                indexes.setAbs(h, it.index);
    2171                                        continue;
    2172                                }
    2173                                while (h - l > 1) {
    2174                                        int m = (l + h) / 2;
    2175                                        long vm = values.getLong(m);
    2176                                        if (x < vm) {
    2177                                                h = m;
    2178                                        } else if (x > vm) {
    2179                                                l = m;
    2180                                        } else {
    2181                                                if (indexes.getAbs(m) < 0)
    2182                                                        indexes.setAbs(m, it.index);
    2183                                                break;
    2184                                        }
    2185                                }
    2186                        }
    2187                } else {
    2188                        while (it.hasNext()) {
    2189                                double x = a.getElementDoubleAbs(it.index);
    2190
    2191                                int l = 0; // binary search to find value in sorted dataset
    2192                                double vl = values.getDouble(l);
    2193                                if (x <= vl) {
    2194                                        if (x == vl && indexes.getAbs(l) < 0)
    2195                                                indexes.setAbs(l, it.index);
    2196                                        continue;
    2197                                }
    2198                                int h = n - 1;
    2199                                double vh = values.getDouble(h);
    2200                                if (x >= vh) {
    2201                                        if (x == vh && indexes.getAbs(h) < 0)
    2202                                                indexes.setAbs(h, it.index);
    2203                                        continue;
    2204                                }
    2205                                while (h - l > 1) {
    2206                                        int m = (l + h) / 2;
    2207                                        double vm = values.getDouble(m);
    2208                                        if (x < vm) {
    2209                                                h = m;
    2210                                        } else if (x > vm) {
    2211                                                l = m;
    2212                                        } else {
    2213                                                if (indexes.getAbs(m) < 0)
    2214                                                        indexes.setAbs(m, it.index);
    2215                                                break;
    2216                                        }
    2217                                }
    2218                        }
    2219                }
    2220                return indexes;
    2221        }
    2222
    2223        /**
    2224         * Find indexes in sorted dataset of values for each value in other dataset
    2225         * @param a
    2226         * @param values sorted 1D dataset of values to find
    2227         * @return absolute indexes of values (-1 is used to indicate value not found)
    2228         */
    2229        public static IntegerDataset findIndexesForValues(final Dataset a, final Dataset values) {
    2230                if (values.getRank() != 1) {
    2231                        throw new IllegalArgumentException("Values dataset must be 1D");
    2232                }
    2233                IntegerDataset indexes = new IntegerDataset(a.getSize());
    2234                indexes.fill(-1);
    2235
    2236                IndexIterator it = a.getIterator();
    2237                int i = -1;
    2238                final int n = values.getSize();
    2239                if (values instanceof LongDataset) {
    2240                        while (it.hasNext()) {
    2241                                i++;
    2242                                long x = a.getElementLongAbs(it.index);
    2243
    2244                                int l = 0; // binary search to find value in sorted dataset
    2245                                long vl = values.getLong(l);
    2246                                if (x <= vl) {
    2247                                        if (x == vl)
    2248                                                indexes.setAbs(i, l);
    2249                                        continue;
    2250                                }
    2251                                int h = n - 1;
    2252                                long vh = values.getLong(h);
    2253                                if (x >= vh) {
    2254                                        if (x == vh)
    2255                                                indexes.setAbs(i, h);
    2256                                        continue;
    2257                                }
    2258                                while (h - l > 1) {
    2259                                        int m = (l + h) / 2;
    2260                                        long vm = values.getLong(m);
    2261                                        if (x < vm) {
    2262                                                h = m;
    2263                                        } else if (x > vm) {
    2264                                                l = m;
    2265                                        } else {
    2266                                                indexes.setAbs(i, m);
    2267                                                break;
    2268                                        }
    2269                                }
    2270                        }
    2271                } else {
    2272                        while (it.hasNext()) {
    2273                                i++;
    2274                                double x = a.getElementDoubleAbs(it.index);
    2275
    2276                                int l = 0; // binary search to find value in sorted dataset
    2277                                double vl = values.getDouble(l);
    2278                                if (x <= vl) {
    2279                                        if (x == vl)
    2280                                                indexes.setAbs(i, l);
    2281                                        continue;
    2282                                }
    2283                                int h = n - 1;
    2284                                double vh = values.getDouble(h);
    2285                                if (x >= vh) {
    2286                                        if (x == vh)
    2287                                                indexes.setAbs(i, h);
    2288                                        continue;
    2289                                }
    2290                                while (h - l > 1) {
    2291                                        int m = (l + h) / 2;
    2292                                        double vm = values.getDouble(m);
    2293                                        if (x < vm) {
    2294                                                h = m;
    2295                                        } else if (x > vm) {
    2296                                                l = m;
    2297                                        } else {
    2298                                                indexes.setAbs(i, m);
    2299                                                break;
    2300                                        }
    2301                                }
    2302                        }
    2303                }
    2304
    2305                return indexes;
    2306        }
    2307
    2308        /**
    2309         * Roll items over given axis by given amount
    2310         * @param a
    2311         * @param shift
    2312         * @param axis if null, then roll flattened dataset
    2313         * @return rolled dataset
    2314         */
    2315        public static <T extends Dataset> T roll(final T a, final int shift, Integer axis) {
    2316                Dataset r = DatasetFactory.zeros(a);
    2317                int is = a.getElementsPerItem();
    2318                if (axis == null) {
    2319                        IndexIterator it = a.getIterator();
    2320                        int s = r.getSize();
    2321                        int i = shift % s;
    2322                        if (i < 0)
    2323                                i += s;
    2324                        while (it.hasNext()) {
    2325                                r.setObjectAbs(i, a.getObjectAbs(it.index));
    2326                                i += is;
    2327                                if (i >= s) {
    2328                                        i %= s;
    2329                                }
    2330                        }
    2331                } else {
    2332                        axis = a.checkAxis(axis);
    2333                        PositionIterator pi = a.getPositionIterator(axis);
    2334                        int s = a.getShapeRef()[axis];
    2335                        Dataset u = DatasetFactory.zeros(is, a.getClass(), new int[] {s});
    2336                        Dataset v = DatasetFactory.zeros(u);
    2337                        int[] pos = pi.getPos();
    2338                        boolean[] hit = pi.getOmit();
    2339                        while (pi.hasNext()) {
    2340                                a.copyItemsFromAxes(pos, hit, u);
    2341                                int i = shift % s;
    2342                                if (i < 0)
    2343                                        i += s;
    2344                                for (int j = 0; j < s; j++) {
    2345                                        v.setObjectAbs(i, u.getObjectAbs(j*is));
    2346                                        i += is;
    2347                                        if (i >= s) {
    2348                                                i %= s;
    2349                                        }
    2350                                }
    2351                                r.setItemsOnAxes(pos, hit, v.getBuffer());
    2352                        }
    2353                }
    2354                return (T) r;
    2355        }
    2356
    2357        /**
    2358         * Roll the specified axis backwards until it lies in given position
    2359         * @param a
    2360         * @param axis The rolled axis (index in shape array). Other axes are left unchanged in relative positions
    2361         * @param start The position with it right of the destination of the rolled axis
    2362         * @return dataset with rolled axis
    2363         */
    2364        public static <T extends Dataset> T rollAxis(final T a, int axis, int start) {
    2365                int r = a.getRank();
    2366                axis = a.checkAxis(axis);
    2367                if (start < 0)
    2368                        start += r;
    2369                if (start < 0 || start > r) {
    2370                        throw new IllegalArgumentException("Start is out of range: it should be >= 0 and <= " + r);
    2371                }
    2372                if (axis < start)
    2373                        start--;
    2374
    2375                if (axis == start)
    2376                        return a;
    2377
    2378                ArrayList<Integer> axes = new ArrayList<Integer>();
    2379                for (int i = 0; i < r; i++) {
    2380                        if (i != axis) {
    2381                                axes.add(i);
    2382                        }
    2383                }
    2384                axes.add(start, axis);
    2385                int[] aa = new int[r];
    2386                for (int i = 0; i < r; i++) {
    2387                        aa[i] = axes.get(i);
    2388                }
    2389                return (T) a.getTransposedView(aa);
    2390        }
    2391
    2392        private static SliceND createFlippedSlice(final Dataset a, int axis) {
    2393                int[] shape = a.getShapeRef();
    2394                SliceND slice = new SliceND(shape);
    2395                slice.flip(axis);
    2396                return slice;
    2397        }
    2398
    2399        /**
    2400         * Flip items in left/right direction, column-wise, or along second axis
    2401         * @param a dataset must be at least 2D
    2402         * @return view of flipped dataset
    2403         */
    2404        public static <T extends Dataset> T flipLeftRight(final T a) {
    2405                if (a.getRank() < 2) {
    2406                        throw new IllegalArgumentException("Dataset must be at least 2D");
    2407                }
    2408                return (T) a.getSliceView(createFlippedSlice(a, 1));
    2409        }
    2410
    2411        /**
    2412         * Flip items in up/down direction, row-wise, or along first axis
    2413         * @param a dataset
    2414         * @return view of flipped dataset
    2415         */
    2416        public static <T extends Dataset> T flipUpDown(final T a) {
    2417                return (T) a.getSliceView(createFlippedSlice(a, 0));
    2418        }
    2419
    2420        /**
    2421         * Rotate items in first two dimension by 90 degrees anti-clockwise
    2422         * @param a dataset must be at least 2D
    2423         * @return view of flipped dataset
    2424         */
    2425        public static <T extends Dataset> T rotate90(final T a) {
    2426                return rotate90(a, 1);
    2427        }
    2428
    2429        /**
    2430         * Rotate items in first two dimension by 90 degrees anti-clockwise
    2431         * @param a dataset must be at least 2D
    2432         * @param k number of 90-degree rotations
    2433         * @return view of flipped dataset
    2434         */
    2435        public static <T extends Dataset> T rotate90(final T a, int k) {
    2436                k = k % 4;
    2437                while (k < 0) {
    2438                        k += 4;
    2439                }
    2440                int r = a.getRank();
    2441                if (r < 2) {
    2442                        throw new IllegalArgumentException("Dataset must be at least 2D");
    2443                }
    2444                switch (k) {
    2445                case 1: case 3:
    2446                        int[] axes = new int[r];
    2447                        axes[0] = 1;
    2448                        axes[1] = 0;
    2449                        for (int i = 2; i < r; i++) {
    2450                                axes[i] = i;
    2451                        }
    2452                        Dataset t = a.getTransposedView(axes);
    2453                        return (T) t.getSliceView(createFlippedSlice(t, k == 1 ? 0 : 1));
    2454                case 2:
    2455                        SliceND s = createFlippedSlice(a, 0);
    2456                        s.flip(1);
    2457                        return (T) a.getSliceView(s);
    2458                default:
    2459                case 0:
    2460                        return a;
    2461                }
    2462        }
    2463
    2464        /**
    2465         * Select content according where condition is true. All inputs are broadcasted to a maximum shape
    2466         * @param condition boolean dataset
    2467         * @param x
    2468         * @param y
    2469         * @return dataset where content is x or y depending on whether condition is true or otherwise
    2470         */
    2471        public static Dataset select(BooleanDataset condition, Object x, Object y) {
    2472                Object[] all = new Object[] {condition, x, y};
    2473                Dataset[] dAll = BroadcastUtils.convertAndBroadcast(all);
    2474                condition = (BooleanDataset) dAll[0];
    2475                Dataset dx = dAll[1];
    2476                Dataset dy = dAll[2];
    2477                Class<? extends Dataset> dc = InterfaceUtils.getBestInterface(dx.getClass(), dy.getClass());
    2478                int ds = Math.max(dx.getElementsPerItem(), dy.getElementsPerItem());
    2479
    2480                Dataset r = DatasetFactory.zeros(ds, dc, condition.getShapeRef());
    2481                IndexIterator iter = condition.getIterator(true);
    2482                final int[] pos = iter.getPos();
    2483                int i = 0;
    2484                while (iter.hasNext()) {
    2485                        r.setObjectAbs(i++, condition.getElementBooleanAbs(iter.index) ? dx.getObject(pos) : dy.getObject(pos));
    2486                }
    2487                return r;
    2488        }
    2489
    2490        /**
    2491         * Select content from choices where condition is true, otherwise use default. All inputs are broadcasted to a maximum shape
    2492         * @param conditions array of boolean datasets
    2493         * @param choices array of datasets or objects
    2494         * @param def default value (can be a dataset)
    2495         * @return dataset
    2496         */
    2497        public static Dataset select(BooleanDataset[] conditions, Object[] choices, Object def) {
    2498                final int n = conditions.length;
    2499                if (choices.length != n) {
    2500                        throw new IllegalArgumentException("Choices list is not same length as conditions list");
    2501                }
    2502                Object[] all = new Object[2*n];
    2503                System.arraycopy(conditions, 0, all, 0, n);
    2504                System.arraycopy(choices, 0, all, n, n);
    2505                Dataset[] dAll = BroadcastUtils.convertAndBroadcast(all);
    2506                conditions = new BooleanDataset[n];
    2507                Dataset[] dChoices = new Dataset[n];
    2508                System.arraycopy(dAll, 0, conditions, 0, n);
    2509                System.arraycopy(dAll, n, dChoices, 0, n);
    2510                Class<? extends Dataset> dc = null;
    2511                int ds = -1;
    2512                for (int i = 0; i < n; i++) {
    2513                        Dataset a = dChoices[i];
    2514                        Class<? extends Dataset> c = a.getClass();
    2515                        dc = InterfaceUtils.getBestInterface(dc, c);
    2516                        int s = a.getElementsPerItem();
    2517                        if (s > ds) {
    2518                                ds = s;
    2519                }
    2520                }
    2521                if (dc == null || ds < 1) {
    2522                        throw new IllegalArgumentException("Dataset types of choices are invalid");
    2523                }
    2524
    2525                Dataset r = DatasetFactory.zeros(ds, dc, conditions[0].getShapeRef());
    2526                Dataset d = DatasetFactory.createFromObject(def).getBroadcastView(r.getShapeRef());
    2527                PositionIterator iter = new PositionIterator(r.getShapeRef());
    2528                final int[] pos = iter.getPos();
    2529                int i = 0;
    2530                while (iter.hasNext()) {
    2531                        int j = 0;
    2532                        for (; j < n; j++) {
    2533                                if (conditions[j].get(pos)) {
    2534                                        r.setObjectAbs(i++, dChoices[j].getObject(pos));
    2535                                        break;
    2536                                }
    2537                        }
    2538                        if (j == n) {
    2539                                r.setObjectAbs(i++, d.getObject(pos));
    2540                        }
    2541                }
    2542                return r;
    2543        }
    2544
    2545        /**
    2546         * Choose content from choices where condition is true, otherwise use default. All inputs are broadcasted to a maximum shape
    2547         * @param index integer dataset (ideally, items should be in [0, n) range, if there are n choices)
    2548         * @param choices array of datasets or objects
    2549         * @param throwAIOOBE if true, throw array index out of bound exception
    2550         * @param clip true to clip else wrap indices out of bounds; only used when throwAOOBE is false
    2551         * @return dataset
    2552         */
    2553        public static Dataset choose(IntegerDataset index, Object[] choices, boolean throwAIOOBE, boolean clip) {
    2554                final int n = choices.length;
    2555                Object[] all = new Object[n + 1];
    2556                System.arraycopy(choices, 0, all, 0, n);
    2557                all[n] = index;
    2558                Dataset[] dChoices = BroadcastUtils.convertAndBroadcast(all);
    2559                Class<? extends Dataset> dc = null;
    2560                int ds = -1;
    2561                int mr = -1;
    2562                for (int i = 0; i < n; i++) {
    2563                        Dataset a = dChoices[i];
    2564                        int r = a.getRank();
    2565                        if (r > mr) {
    2566                                mr = r;
    2567                        }
    2568                        dc = InterfaceUtils.getBestInterface(dc, a.getClass());
    2569                        int s = a.getElementsPerItem();
    2570                        if (s > ds) {
    2571                                ds = s;
    2572                }
    2573                }
    2574                if (dc == null || ds < 1) {
    2575                        throw new IllegalArgumentException("Dataset types of choices are invalid");
    2576                }
    2577                index = (IntegerDataset) dChoices[n];
    2578                dChoices[n] = null;
    2579
    2580                Dataset r = DatasetFactory.zeros(ds, dc, index.getShapeRef());
    2581                IndexIterator iter = index.getIterator(true);
    2582                final int[] pos = iter.getPos();
    2583                int i = 0;
    2584                while (iter.hasNext()) {
    2585                        int j = index.getAbs(iter.index);
    2586                        if (j < 0) {
    2587                                if (throwAIOOBE)
    2588                                        throw new ArrayIndexOutOfBoundsException(j);
    2589                                if (clip) {
    2590                                        j = 0;
    2591                                } else {
    2592                                        j %= n;
    2593                                        j += n; // as remainder still negative
    2594                                }
    2595                        }
    2596                        if (j >= n) {
    2597                                if (throwAIOOBE)
    2598                                        throw new ArrayIndexOutOfBoundsException(j);
    2599                                if (clip) {
    2600                                        j = n - 1;
    2601                                } else {
    2602                                        j %= n;
    2603                                }
    2604                        }
    2605                        Dataset c = dChoices[j];
    2606                        r.setObjectAbs(i++, c.getObject(pos));
    2607                }
    2608                return r;
    2609        }
    2610
    2611        /**
    2612         * Calculate positions in given shape from a dataset of 1-D indexes
    2613         * @param indices
    2614         * @param shape
    2615         * @return list of positions as integer datasets
    2616         */
    2617        public static List<IntegerDataset> calcPositionsFromIndexes(Dataset indices, int[] shape) {
    2618                int rank = shape.length;
    2619                List<IntegerDataset> posns = new ArrayList<IntegerDataset>();
    2620                int[] iShape = indices.getShapeRef();
    2621                for (int i = 0; i < rank; i++) {
    2622                        posns.add(new IntegerDataset(iShape));
    2623                }
    2624                IndexIterator it = indices.getIterator(true);
    2625                int[] pos = it.getPos();
    2626                while (it.hasNext()) {
    2627                        int n = indices.getInt(pos);
    2628                        int[] p = ShapeUtils.getNDPositionFromShape(n, shape);
    2629                        for (int i = 0; i < rank; i++) {
    2630                                posns.get(i).setItem(p[i], pos);
    2631                        }
    2632                }
    2633                return posns;
    2634        }
    2635
    2636
    2637        /**
    2638         * Calculate indexes in given shape from datasets of position
    2639         * @param positions as a list of datasets where each holds the position in a dimension
    2640         * @param shape
    2641         * @param mode either null, zero-length, unit length or length of rank of shape where
    2642         *  0 = raise exception, 1 = wrap, 2 = clip
    2643         * @return indexes as an integer dataset
    2644         */
    2645        public static IntegerDataset calcIndexesFromPositions(List<? extends Dataset> positions, int[] shape, int... mode) {
    2646                int rank = shape.length;
    2647                if (positions.size() != rank) {
    2648                        throw new IllegalArgumentException("Number of position datasets must be equal to rank of shape");
    2649                }
    2650
    2651                if (mode == null || mode.length == 0) {
    2652                        mode = new int[rank];
    2653                } else if (mode.length == 1) {
    2654                        int m = mode[0];
    2655                        mode = new int[rank];
    2656                        Arrays.fill(mode, m);
    2657                } else if (mode.length != rank) {
    2658                        throw new IllegalArgumentException("Mode length greater than one must match rank of shape");
    2659                }
    2660                for (int i = 0; i < rank; i++) {
    2661                        int m = mode[i];
    2662                        if (m < 0 || m > 2) {
    2663                                throw new IllegalArgumentException("Unknown mode value - it must be 0, 1, or 2");
    2664                        }
    2665                }
    2666
    2667                Dataset p = positions.get(0);
    2668                IntegerDataset indexes = new IntegerDataset(p.getShapeRef());
    2669                IndexIterator it = p.getIterator(true);
    2670                int[] iPos = it.getPos();
    2671                int[] tPos = new int[rank];
    2672                while (it.hasNext()) {
    2673                        for (int i = 0; i < rank; i++) {
    2674                                p = positions.get(i);
    2675                                int j = p.getInt(iPos);
    2676                                int d = shape[i];
    2677                                if (mode[i] == 0) {
    2678                                        if (j < 0 || j >= d) {
    2679                                                throw new ArrayIndexOutOfBoundsException("Position value exceeds dimension in shape");
    2680                                        }
    2681                                } else if (mode[i] == 1) {
    2682                                        while (j < 0)
    2683                                                j += d;
    2684                                        while (j >= d)
    2685                                                j -= d;
    2686                                } else {
    2687                                        if (j < 0)
    2688                                                j = 0;
    2689                                        if (j >= d)
    2690                                                j = d - 1;
    2691                                }
    2692                                tPos[i] = j;
    2693                        }
    2694                        indexes.set(ShapeUtils.getFlat1DIndex(shape, tPos), iPos);
    2695                }
    2696
    2697                return indexes;
    2698        }
    2699
    2700        /**
    2701         * Serialize dataset by flattening it. Discards metadata
    2702         * @param data
    2703         * @return some java array
    2704         */
    2705        public static Serializable serializeDataset(final IDataset data) {
    2706                Dataset d = convertToDataset(data).getView(false);
    2707                d.clearMetadata(null);
    2708                return d.flatten().getBuffer();
    2709        }
    2710
    2711        /**
    2712         * Extract values where condition is non-zero. This is similar to Dataset#getByBoolean but supports broadcasting
    2713         * @param data
    2714         * @param condition should be broadcastable to data
    2715         * @return 1-D dataset of values
    2716         */
    2717        public static Dataset extract(final IDataset data, final IDataset condition) {
    2718                Dataset a = convertToDataset(data.getSliceView());
    2719                Dataset b = cast(BooleanDataset.class, condition.getSliceView());
    2720
    2721                try {
    2722                        return a.getByBoolean(b);
    2723                } catch (IllegalArgumentException e) {
    2724                        final int length = ((Number) b.sum()).intValue();
    2725
    2726                        BroadcastPairIterator it = new BroadcastPairIterator(a, b, null, false);
    2727                        int size = ShapeUtils.calcSize(it.getShape());
    2728                        Dataset c;
    2729                        if (length < size) {
    2730                                int[] ashape = it.getFirstShape();
    2731                                int[] bshape = it.getSecondShape();
    2732                                int r = ashape.length;
    2733                                size = length;
    2734                                for (int i = 0; i < r; i++) {
    2735                                        int s = ashape[i];
    2736                                        if (s > 1 && bshape[i] == 1) {
    2737                                                size *= s;
    2738                                        }
    2739                                }
    2740                        }
    2741                        c = DatasetFactory.zeros(a.getClass(), size);
    2742
    2743                        int i = 0;
    2744                        if (it.isOutputDouble()) {
    2745                                while (it.hasNext()) {
    2746                                        if (it.bLong != 0) {
    2747                                                c.setObjectAbs(i++, it.aDouble);
    2748                                        }
    2749                                }
    2750                        } else {
    2751                                while (it.hasNext()) {
    2752                                        if (it.bLong != 0) {
    2753                                                c.setObjectAbs(i++, it.aLong);
    2754                                        }
    2755                                }
    2756                        }
    2757
    2758                        return c;
    2759                }
    2760        }
    2761
    2762        /**
    2763         * Set shape to keep original rank
    2764         * @param a
    2765         * @param originalShape
    2766         * @param axes
    2767         * @since 2.2
    2768         */
    2769        public static void setShapeToOriginalRank(ILazyDataset a, int[] originalShape, int... axes) {
    2770                a.setShape(ShapeUtils.getReducedShapeKeepRank(originalShape, axes));
    2771        }
    2772}