    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.annotation.Annotation;
    017import java.lang.reflect.Array;
    018import java.lang.reflect.Field;
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.Collection;
    022import java.util.concurrent.ConcurrentMap;
    023import java.util.concurrent.ConcurrentHashMap;
    024import java.util.List;
    025import java.util.Map;
    026
    027import org.eclipse.january.DatasetException;
    028import org.eclipse.january.MetadataException;
    029import org.eclipse.january.metadata.Dirtiable;
    030import org.eclipse.january.metadata.ErrorMetadata;
    031import org.eclipse.january.metadata.IMetadata;
    032import org.eclipse.january.metadata.MetadataFactory;
    033import org.eclipse.january.metadata.MetadataType;
    034import org.eclipse.january.metadata.Reshapeable;
    035import org.eclipse.january.metadata.Sliceable;
    036import org.eclipse.january.metadata.Transposable;
    037import org.slf4j.Logger;
    038import org.slf4j.LoggerFactory;
    039
    040/**
    041 * Common base for both lazy and normal dataset implementations
    042 */
    043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
    044
    045        private static final long serialVersionUID = 767926846438976050L;
    046
    047        private static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
    048
    049        transient private boolean dirty = true; // indicate dirty state of metadata
    050        protected String name = "";
    051
    052        /**
    053         * The shape or dimensions of the dataset
    054         */
    055        protected int[] shape;
    056
    057        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
    058
    059        /**
    060         * @return type of dataset item
    061         */
    062        abstract public int getDType();
    063
    064        @Override
    065        public LazyDatasetBase clone() {
    066                return null;
    067        }
    068
    069        @Override
    070        public boolean equals(Object obj) {
    071                if (this == obj) {
    072                        return true;
    073                }
    074                if (obj == null) {
    075                        return false;
    076                }
    077                if (!getClass().equals(obj.getClass())) {
    078                        return false;
    079                }
    080
    081                LazyDatasetBase other = (LazyDatasetBase) obj;
    082                if (getDType() != other.getDType()) {
    083                        return false;
    084                }
    085                if (getElementsPerItem() != other.getElementsPerItem()) {
    086                        return false;
    087                }
    088                if (!Arrays.equals(shape, other.shape)) {
    089                        return false;
    090                }
    091                return true;
    092        }
    093
    094        @Override
    095        public int hashCode() {
    096                int hash = getDType() * 17 + getElementsPerItem();
    097                int rank = shape.length;
    098                for (int i = 0; i < rank; i++) {
    099                        hash = hash*17 + shape[i];
    100                }
    101                return hash;
    102        }
    103
    104        @Override
    105        public String getName() {
    106                return name;
    107        }
    108
    109        @Override
    110        public void setName(String name) {
    111                this.name = name;
    112        }
    113
    114        @Override
    115        public int[] getShape() {
    116                return shape.clone();
    117        }
    118
    119        @Override
    120        public int getRank() {
    121                return shape.length;
    122        }
    123
    124        /**
    125         * This method allows anything that dirties the dataset to clear various metadata values
    126         * so that the other methods can work correctly.
    127         * @since 2.1
    128         */
    129        public void setDirty() {
    130                dirty = true;
    131        }
    132
    133        protected void checkSliceND(SliceND slice) {
    134                if (slice != null) {
    135                        int[] source = slice.getSourceShape();
    136                        boolean fail = false;
    137                        if (slice.isExpanded()) {
    138                                fail = shape.length != source.length;
    139                                if (!fail) {
    140                                        for (int i = 0; i < shape.length; i++) {
    141                                                if (shape[i] > source[i]) {
    142                                                        fail = true;
    143                                                        break;
    144                                                }
    145                                        }
    146                                }
    147                        } else {
    148                                fail = !Arrays.equals(shape, source);
    149                        }
    150                        if (fail) {
    151                                throw new IllegalArgumentException("Slice's shape must match dataset's shape");
    152                        }
    153                }
    154        }
    155
    156        /**
    157         * Find first sub-interface of (or class that directly implements) MetadataType
    158         * @param clazz metadata type
    159         * @return sub-interface
    160         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
    161         */
    162        @SuppressWarnings("unchecked")
    163        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
    164                if (clazz.equals(MetadataType.class)) {
    165                        throw new IllegalArgumentException("Cannot accept MetadataType");
    166                }
    167
    168                if (clazz.isInterface()) {
    169                        return clazz;
    170                }
    171
    172                if (clazz.isAnonymousClass()) { // special case
    173                        Class<?> s = clazz.getSuperclass();
    174                        if (!s.equals(Object.class)) {
    175                                // only use super class if it is not an anonymous class of an interface
    176                                clazz = (Class<? extends MetadataType>) s;
    177                        }
    178                }
    179
    180                for (Class<?> c : clazz.getInterfaces()) {
    181                        if (c.equals(MetadataType.class)) {
    182                                if (clazz.isAnonymousClass()) {
    183                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
    184                                }
    185                                return clazz;
    186                        }
    187                        if (MetadataType.class.isAssignableFrom(c)) {
    188                                return (Class<? extends MetadataType>) c;
    189                        }
    190                }
    191
    192                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
    193                if (c != null) {
    194                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
    195                }
    196
    197                logger.error("Somehow the search for metadata type interface ended in a bad place");
    198                assert false; // should not be able to get here!!!
    199                return null;
    200        }
    201
    202        @Override
    203        public void setMetadata(MetadataType metadata) {
    204                addMetadata(metadata, true);
    205        }
    206
    207        @Override
    208        public void addMetadata(MetadataType metadata) {
    209                addMetadata(metadata, false);
    210        }
    211
    212        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
    213                if (metadata == null) {
    214                        return;
    215                }
    216
    217                if (this.metadata == null) {
    218                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    219                }
    220
    221                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
    222                if (!this.metadata.containsKey(clazz)) {
    223                        this.metadata.put(clazz, new ArrayList<MetadataType>());
    224                } else if (clear) {
    225                        this.metadata.get(clazz).clear();
    226                }
    227                this.metadata.get(clazz).add(metadata);
    228
    229                // add for special case of sub-interfaces of IMetadata
    230                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
    231                        clazz = IMetadata.class;
    232                        if (!this.metadata.containsKey(clazz)) {
    233                                this.metadata.put(clazz, new ArrayList<MetadataType>());
    234                        } else if (clear) {
    235                                this.metadata.get(clazz).clear();
    236                        }
    237                        this.metadata.get(clazz).add(metadata);
    238                }
    239        }
    240
    241        @Override
    242        @Deprecated
    243        public synchronized IMetadata getMetadata() {
    244                return getFirstMetadata(IMetadata.class);
    245        }
    246
    247        @SuppressWarnings("unchecked")
    248        @Override
    249        public synchronized <T extends MetadataType> List<T> getMetadata(Class<T> clazz) throws MetadataException {
    250                if (metadata == null) {
    251                        dirty = false;
    252                        return null;
    253                }
    254
    255                if (dirty) {
    256                        dirtyMetadata();
    257                        dirty = false;
    258                }
    259
    260                if (clazz == null) {
    261                        List<T> all = new ArrayList<>();
    262                        for (Class<? extends MetadataType> c : metadata.keySet()) {
    263                                all.addAll((Collection<T>) metadata.get(c));
    264                        }
    265                        return all;
    266                }
    267
    268                return (List<T>) metadata.get(findMetadataTypeSubInterfaces(clazz));
    269        }
    270
    271        @Override
    272        public synchronized <T extends MetadataType> T getFirstMetadata(Class<T> clazz) {
    273                try {
    274                        List<T> ml = getMetadata(clazz);
    275                        if (ml == null) {
    276                                return null;
    277                        }
    278                        for (T t : ml) {
    279                                if (clazz.isInstance(t)) {
    280                                        return t;
    281                                }
    282                        }
    283                } catch (Exception e) {
    284                        logger.error("Get metadata failed!",e);
    285                }
    286
    287                return null;
    288        }
    289
    290        @Override
    291        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
    292                if (metadata == null) {
    293                        return;
    294                }
    295
    296                if (clazz == null) {
    297                        metadata.clear();
    298                        return;
    299                }
    300
    301                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
    302                if( list != null) {
    303                        list.clear();
    304                }
    305        }
    306
    307        /**
    308         * @return copy of metadata
    309         * @since 2.0
    310         */
    311        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
    312                return copyMetadata(metadata);
    313        }
    314
    315        /**
    316         * @param metadata type
    317         * @return copy of metadata of given type
    318         * @since 2.0
    319         */
    320        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
    321                if (metadata == null) {
    322                        return null;
    323                }
    324
    325                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    326                copyMetadata(metadata, map);
    327                return map;
    328        }
    329
    330        private static void copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> inMetadata,
    331                        Map<Class<? extends MetadataType>, List<MetadataType>> outMetadata) {
    332                for (Class<? extends MetadataType> c : inMetadata.keySet()) {
    333                        List<MetadataType> l = inMetadata.get(c);
    334                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
    335                        outMetadata.put(c, nl);
    336                        for (MetadataType m : l) {
    337                                if (m == null || isMetadataDirty(m)) { // skip dirty metadata
    338                                        continue;
    339                                }
    340                                nl.add(m.clone());
    341                        }
    342                }
    343        }
    344
    345        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
    346                copyMetadata(oldMetadata, metadata);
    347        }
    348
    349        /**
    350         * @param a dataset
    351         * @param clone if true, copy metadata
    352         * @return copy of metadata
    353         * @since 2.2
    354         */
    355        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(ILazyDataset a, boolean clone) {
    356                List<MetadataType> all = null;
    357                try {
    358                        all = a.getMetadata(null);
    359                } catch (Exception e) {
    360                }
    361                if (all == null) {
    362                        return null;
    363                }
    364
    365                ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    366
    367                for (MetadataType m : all) {
    368                        if (m == null || isMetadataDirty(m)) { // skip dirty metadata
    369                                continue;
    370                        }
    371                        Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass());
    372                        List<MetadataType> l = map.get(c);
    373                        if (l == null) {
    374                                l = new ArrayList<MetadataType>();
    375                                map.put(c, l);
    376                        }
    377                        if (clone) {
    378                                m = m.clone();
    379                        }
    380                        l.add(m);
    381                }
    382                return map;
    383        }
    384
    385        private static boolean isMetadataDirty(MetadataType m) {
    386                Class<? extends MetadataType> c = m.getClass();
    387                for (Field f : c.getDeclaredFields()) {
    388                        if (f.isAnnotationPresent(Dirtiable.class)) {
    389                                Class<?> t = f.getType();
    390                                if (t.equals(boolean.class) || t.equals(Boolean.class)) {
    391                                        try {
    392                                                f.setAccessible(true);
    393                                                Object o = f.get(m);
    394                                                if (o.equals(true)) {
    395                                                        return true;
    396                                                }
    397                                        } catch (Exception e) {
    398                                                logger.debug("Could not retrieve value of dirty variable: {}", c.getCanonicalName(), e);
    399                                        }
    400                                }
    401                        }
    402                }
    403
    404                return false;
    405        }
    406
    407        interface MetadatasetAnnotationOperation {
    408                /**
    409                 * Process value of given field
    410                 * <p>
    411                 * When the field is not a container then the returned value
    412                 * may replace the old value
    413                 * @param f given field
    414                 * @param o value of field
    415                 * @return transformed field
    416                 */
    417                Object processField(Field f, Object o);
    418
    419                /**
    420                 * @return annotated class
    421                 */
    422                Class<? extends Annotation> getAnnClass();
    423
    424                /**
    425                 * @param axis
    426                 * @return number of dimensions to insert or remove
    427                 */
    428                int change(int axis);
    429
    430                /**
    431                 *
    432                 * @return rank or -1 to match
    433                 */
    434                int getNewRank();
    435
    436                /**
    437                 * Run on given lazy dataset
    438                 * @param lz
    439                 * @return
    440                 */
    441                ILazyDataset run(ILazyDataset lz);
    442        }
    443
    444        class MdsSlice implements MetadatasetAnnotationOperation {
    445                private boolean asView;
    446                private SliceND slice;
    447                private int[] oShape;
    448                private long oSize;
    449
    450                public MdsSlice(boolean asView, SliceND slice) {
    451                        this.asView = asView;
    452                        this.slice = slice;
    453                        oShape = slice.getSourceShape();
    454                        oSize = ShapeUtils.calcLongSize(oShape);
    455                }
    456
    457                @Override
    458                public Object processField(Field field, Object o) {
    459                        return o;
    460                }
    461
    462                @Override
    463                public Class<? extends Annotation> getAnnClass() {
    464                        return Sliceable.class;
    465                }
    466
    467                @Override
    468                public int change(int axis) {
    469                        return 0;
    470                }
    471
    472                @Override
    473                public int getNewRank() {
    474                        return -1;
    475                }
    476
    477                @Override
    478                public ILazyDataset run(ILazyDataset lz) {
    479                        int rank = lz.getRank();
    480                        if (slice.getStart().length != rank) {
    481                                throw new IllegalArgumentException("Slice rank does not match dataset!");
    482                        }
    483
    484                        int[] shape = lz.getShape();
    485                        SliceND nslice;
    486                        if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) {
    487                                nslice = new SliceND(shape);
    488                                for (int i = 0; i < rank; i++) {
    489                                        int s = shape[i];
    490                                        int os = oShape[i];
    491                                        if (s >= os) {
    492                                                nslice.setSlice(i, 0, os, 1);
    493                                        } else if (s == 1) {
    494                                                nslice.setSlice(i, 0, 1, 1);
    495                                        } else {
    496                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
    497                                        }
    498                                }
    499                                lz = lz.getSliceView(nslice);
    500                                shape = nslice.getShape();
    501                        }
    502                        if (lz.getSize() == oSize && Arrays.equals(shape, oShape)) {
    503                                nslice = slice;
    504                        } else {
    505                                nslice = slice.clone();
    506                                for (int i = 0; i < rank; i++) {
    507                                        int s = shape[i];
    508                                        if (s >= oShape[i]) {
    509                                                continue;
    510                                        } else if (s == 1) {
    511                                                nslice.setSlice(i, 0, 1, 1);
    512                                        } else {
    513                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
    514                                        }
    515                                }
    516                                nslice.updateSourceShape(shape);
    517                        }
    518
    519                        if (asView || (lz instanceof IDataset)) {
    520                                return lz.getSliceView(nslice);
    521                        }
    522                        try {
    523                                return lz.getSlice(nslice);
    524                        } catch (DatasetException e) {
    525                                logger.error("Could not slice dataset in metadata", e);
    526                                return null;
    527                        }
    528                }
    529        }
    530
    531        class MdsReshape implements MetadatasetAnnotationOperation {
    532                private boolean matchRank;
    533                private int[] oldShape;
    534                private int[] newShape;
    535                boolean onesOnly;
    536                int[] differences;
    537
    538                /*
    539                 * if only ones then record differences (insertions and deletions)
    540                 *
    541                 * if shape changing, find broadcasted dimensions and disallow
    542                 * merging that include those dimensions
    543                 */
    544                public MdsReshape(final int[] oldShape, final int[] newShape) {
    545                        this.oldShape = oldShape;
    546                        this.newShape = newShape;
    547                        differences = null;
    548                }
    549
    550                @Override
    551                public Object processField(Field field, Object o) {
    552                        Annotation a = field.getAnnotation(Reshapeable.class);
    553                        if (a != null) { // cannot be null
    554                                matchRank = ((Reshapeable) a).matchRank();
    555                        }
    556                        return o;
    557                }
    558
    559                @Override
    560                public Class<? extends Annotation> getAnnClass() {
    561                        return Reshapeable.class;
    562                }
    563
    564                @Override
    565                public int change(int axis) {
    566                        if (matchRank) {
    567                                if (differences == null) {
    568                                        init();
    569                                }
    570
    571                                if (onesOnly) {
    572                                        return differences == null ? 0 : differences[axis];
    573                                }
    574                                throw new UnsupportedOperationException("TODO support other shape operations");
    575                        }
    576                        return 0;
    577                }
    578
    579                @Override
    580                public int getNewRank() {
    581                        return matchRank ? newShape.length : -1;
    582                }
    583
    584                private void init() {
    585                        int or = oldShape.length - 1;
    586                        int nr = newShape.length - 1;
    587                        if (or < 0 || nr < 0) { // zero-rank shapes
    588                                onesOnly = true;
    589                                differences = new int[1];
    590                                differences[0] = or < 0 ? nr + 1 : or + 1;
    591                                return;
    592                        }
    593                        onesOnly = ShapeUtils.differsByOnes(oldShape, newShape);
    594                        int ob = 0;
    595                        int nb = 0;
    596                        if (onesOnly) {
    597                                differences = ShapeUtils.calcShapePadding(oldShape, newShape);
    598                        } else {
    599                                differences = new int[or + 2];
    600                                if (matchRank) {
    601                                        logger.error("Combining dimensions is currently not supported");
    602                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
    603                                }
    604                                // work out mapping: contiguous dimensions can be grouped or split
    605                                while (ob <= or && nb <= nr) {
    606                                        int ol = oldShape[ob];
    607                                        while (ol == 1 && ol <= or) {
    608                                                ob++;
    609                                                ol = oldShape[ob];
    610                                        }
    611                                        int oe = ob + 1;
    612                                        int nl = newShape[nb];
    613                                        while (nl == 1 && nl <= nr) {
    614                                                nb++;
    615                                                nl = newShape[nb];
    616                                        }
    617                                        int ne = nb + 1;
    618                                        if (ol < nl) {
    619                                                differences[ob] = 1;
    620                                                do { // case where new shape combines several dimensions into one dimension
    621                                                        if (oe == (or + 1)) {
    622                                                                break;
    623                                                        }
    624                                                        differences[oe] = 1;
    625                                                        ol *= oldShape[oe++];
    626                                                } while (ol < nl);
    627                                                differences[oe - 1] = oe - ob; // signal end with difference
    628                                                if (nl != ol) {
    629                                                        logger.error("Single dimension is incompatible with subshape");
    630                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    631                                                }
    632                                        } else if (ol > nl) {
    633                                                do { // case where new shape spreads single dimension over several dimensions
    634                                                        if (ne == (nr + 1)) {
    635                                                                break;
    636                                                        }
    637                                                        nl *= newShape[ne++];
    638                                                } while (nl < ol);
    639                                                if (nl != ol) {
    640                                                        logger.error("Subshape is incompatible with single dimension");
    641                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    642                                                }
    643                                        }
    644
    645                                        ob = oe;
    646                                        nb = ne;
    647                                }
    648                        }
    649                }
    650
    651                @Override
    652                public ILazyDataset run(ILazyDataset lz) {
    653                        if (differences == null) {
    654                                init();
    655                        }
    656
    657                        int[] lshape = lz.getShape();
    658                        if (Arrays.equals(newShape, lshape)) {
    659                                return lz;
    660                        }
    661                        int or = lshape.length;
    662                        int nr = newShape.length;
    663                        int[] nshape;
    664                        if (onesOnly) {
    665                                nshape = ShapeUtils.padShape(differences, nr, lshape);
    666                        } else {
    667                                nshape = new int[nr];
    668                                boolean[] broadcast = new boolean[or];
    669                                for (int ob = 0; ob < or; ob++) {
    670                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
    671                                }
    672                                int osize = lz.getSize();
    673
    674                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
    675                                int ob = 0;
    676                                int nsize = 1;
    677                                for (int i = 0; i < nr; i++) {
    678                                        if (ob < or && broadcast[ob]) {
    679                                                if (differences[ob] != 0) {
    680                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
    681                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
    682                                                }
    683                                                nshape[i] = 1;
    684                                        } else {
    685                                                nshape[i] = nsize < osize ? newShape[i] : 1;
    686                                        }
    687                                        nsize *= nshape[i];
    688                                        ob++;
    689                                }
    690                        }
    691
    692                        ILazyDataset nlz;
    693                        if (lz instanceof Dataset) {
    694                                nlz = ((Dataset) lz).reshape(nshape);
    695                        } else {
    696                                nlz = lz.getSliceView();
    697                                nlz.setShape(nshape);
    698                        }
    699                        return nlz;
    700                }
    701        }
    702
    703        class MdsTranspose implements MetadatasetAnnotationOperation {
    704                int[] map;
    705
    706                public MdsTranspose(final int[] axesMap) {
    707                        map = axesMap;
    708                }
    709
    710                @SuppressWarnings({ "rawtypes", "unchecked" })
    711                @Override
    712                public Object processField(Field f, Object o) {
    713                        // reorder arrays and lists according the axes map
    714                        if (o.getClass().isArray()) {
    715                                int l = Array.getLength(o);
    716                                if (l == map.length) {
    717                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    718                                        for (int i = 0; i < l; i++) {
    719                                                Array.set(narray, i, Array.get(o, map[i]));
    720                                        }
    721                                        for (int i = 0; i < l; i++) {
    722                                                Array.set(o, i, Array.get(narray, i));
    723                                        }
    724                                }
    725                        } else if (o instanceof List<?>) {
    726                                List list = (List) o;
    727                                int l = list.size();
    728                                if (l == map.length) {
    729                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    730                                        for (int i = 0; i < l; i++) {
    731                                                Array.set(narray, i, list.get(map[i]));
    732                                        }
    733                                        list.clear();
    734                                        for (int i = 0; i < l; i++) {
    735                                                list.add(Array.get(narray, i));
    736                                        }
    737                                }
    738                        }
    739                        return o;
    740                }
    741
    742                @Override
    743                public Class<? extends Annotation> getAnnClass() {
    744                        return Transposable.class;
    745                }
    746
    747                @Override
    748                public int change(int axis) {
    749                        return 0;
    750                }
    751
    752                @Override
    753                public int getNewRank() {
    754                        return -1;
    755                }
    756
    757                @Override
    758                public ILazyDataset run(ILazyDataset lz) {
    759                        return lz.getTransposedView(map);
    760                }
    761        }
    762
    763        class MdsDirty implements MetadatasetAnnotationOperation {
    764
    765                @Override
    766                public Object processField(Field f, Object o) {
    767                        // throw exception if not boolean???
    768                        Class<?> t = f.getType();
    769                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
    770                                if (o.equals(false)) {
    771                                        o = true;
    772                                }
    773                        }
    774                        return o;
    775                }
    776
    777                @Override
    778                public Class<? extends Annotation> getAnnClass() {
    779                        return Dirtiable.class;
    780                }
    781
    782                @Override
    783                public int change(int axis) {
    784                        return 0;
    785                }
    786
    787                @Override
    788                public int getNewRank() {
    789                        return -1;
    790                }
    791
    792                @Override
    793                public ILazyDataset run(ILazyDataset lz) {
    794                        return lz;
    795                }
    796        }
    797
    798        /**
    799         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
    800         * dataset after cloning the metadata
    801         * @param asView if true then just a view
    802         * @param slice an n-D slice
    803         */
    804        protected void sliceMetadata(boolean asView, final SliceND slice) {
    805                processAnnotatedMetadata(new MdsSlice(asView, slice));
    806        }
    807
    808        /**
    809         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
    810         * or setting the shape
    811         * @param oldShape old shape
    812         * @param newShape new shape
    813         */
    814        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
    815                processAnnotatedMetadata(new MdsReshape(oldShape, newShape));
    816        }
    817
    818        /**
    819         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
    820         * dataset after cloning the metadata
    821         * @param axesMap if zero length then axes order reversed
    822         */
    823        protected void transposeMetadata(final int[] axesMap) {
    824                processAnnotatedMetadata(new MdsTranspose(axesMap));
    825        }
    826
    827        /**
    828         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
    829         * @since 2.0
    830         */
    831        protected void dirtyMetadata() {
    832                processAnnotatedMetadata(new MdsDirty());
    833        }
    834
    835        @SuppressWarnings("unchecked")
    836        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op) {
    837                if (metadata == null)
    838                        return;
    839
    840                for (List<MetadataType> l : metadata.values()) {
    841                        for (MetadataType m : l) {
    842                                if (m == null) {
    843                                        continue;
    844                                }
    845
    846                                Class<? extends MetadataType> mc = m.getClass();
    847                                do { // iterate over super-classes
    848                                        processClass(op, m, mc);
    849                                        Class<?> sclazz = mc.getSuperclass();
    850                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
    851                                                break;
    852                                        }
    853                                        mc = (Class<? extends MetadataType>) sclazz;
    854                                } while (true);
    855                        }
    856                }
    857        }
    858
    859        @SuppressWarnings({ "unchecked", "rawtypes" })
    860        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc) {
    861                for (Field f : mc.getDeclaredFields()) {
    862                        if (!f.isAnnotationPresent(op.getAnnClass()))
    863                                continue;
    864
    865                        try {
    866                                f.setAccessible(true);
    867                                Object o = f.get(m);
    868                                if (o == null) {
    869                                        continue;
    870                                }
    871
    872                                Object no = op.processField(f, o);
    873                                if (no != o) {
    874                                        f.set(m, no);
    875                                        continue;
    876                                }
    877                                Object r = null;
    878                                if (o instanceof ILazyDataset) {
    879                                        try {
    880                                                f.set(m, op.run((ILazyDataset) o));
    881                                        } catch (Exception e) {
    882                                                logger.error("Problem processing " + o, e);
    883                                                throw e;
    884                                        }
    885                                } else if (o.getClass().isArray()) {
    886                                        int l = Array.getLength(o);
    887
    888                                        for (int i = 0; r == null && i < l; i++) {
    889                                                r = Array.get(o, i);
    890                                        }
    891                                        int n = op.getNewRank();
    892                                        if (r == null) {
    893                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    894                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
    895                                                }
    896                                                continue;
    897                                        }
    898                                        if (n < 0) {
    899                                                n = l;
    900                                        }
    901                                        Object narray = Array.newInstance(r.getClass(), n);
    902                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
    903                                                int c = op.change(i);
    904                                                if (c == 0) {
    905                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
    906                                                } else if (c > 0) {
    907                                                        di += c; // add nulls by skipping forward in destination array
    908                                                } else if (c < 0) {
    909                                                        si -= c; // remove dimensions by skipping forward in source array
    910                                                }
    911                                        }
    912                                        if (n == l) {
    913                                                for (int i = 0; i < l; i++) {
    914                                                        Array.set(o, i, Array.get(narray, i));
    915                                                }
    916                                        } else {
    917                                                f.set(m, narray);
    918                                        }
    919                                } else if (o instanceof List<?>) {
    920                                        List list = (List) o;
    921                                        int l = list.size();
    922
    923                                        for (int i = 0; r == null && i < l; i++) {
    924                                                r = list.get(i);
    925                                        }
    926                                        int n = op.getNewRank();
    927                                        if (r == null) {
    928                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    929                                                        list.clear();
    930                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
    931                                                                list.add(null);
    932                                                        }
    933                                                }
    934                                                continue;
    935                                        }
    936
    937                                        if (n < 0) {
    938                                                n = l;
    939                                        }
    940                                        Object narray = Array.newInstance(r.getClass(), n);
    941                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
    942                                                int c = op.change(i);
    943                                                if (c == 0) {
    944                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
    945                                                } else if (c > 0) {
    946                                                        di += c; // add nulls by skipping forward in destination array
    947                                                } else if (c < 0) {
    948                                                        si -= c; // remove dimensions by skipping forward in source array
    949                                                }
    950                                        }
    951                                        list.clear();
    952                                        for (int i = 0; i < n; i++) {
    953                                                list.add(Array.get(narray, i));
    954                                        }
    955                                } else if (o instanceof Map<?,?>) {
    956                                        Map map = (Map) o;
    957                                        for (Object k : map.keySet()) {
    958                                                map.put(k, processObject(op, map.get(k)));
    959                                        }
    960                                }
    961                        } catch (Exception e) {
    962                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
    963                                throw new RuntimeException(e);
    964                        }
    965                }
    966        }
    967
    968        @SuppressWarnings({ "unchecked", "rawtypes" })
    969        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
    970                if (o == null) {
    971                        return o;
    972                }
    973
    974                if (o instanceof ILazyDataset) {
    975                        try {
    976                                return op.run((ILazyDataset) o);
    977                        } catch (Exception e) {
    978                                logger.error("Problem processing " + o, e);
    979                                throw e;
    980                        }
    981                } else if (o.getClass().isArray()) {
    982                        int l = Array.getLength(o);
    983                        for (int i = 0; i < l; i++) {
    984                                Array.set(o, i, processObject(op, Array.get(o, i)));
    985                        }
    986                } else if (o instanceof List<?>) {
    987                        List list = (List) o;
    988                        for (int i = 0, imax = list.size(); i < imax; i++) {
    989                                list.set(i, processObject(op, list.get(i)));
    990                        }
    991                } else if (o instanceof Map<?,?>) {
    992                        Map map = (Map) o;
    993                        for (Object k : map.keySet()) {
    994                                map.put(k, processObject(op, map.get(k)));
    995                        }
    996                }
    997                return o;
    998        }
    999
    1000        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
    1001                ILazyDataset d = null;
    1002                if (blob instanceof ILazyDataset) {
    1003                        d = (ILazyDataset) blob;
    1004                        if (d instanceof IDataset) {
    1005                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
    1006                                int is = ed.getElementsPerItem();
    1007                                if (is != 1 && is != getElementsPerItem()) {
    1008                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
    1009                                }
    1010                                d = ed.cast(is == 1 ? DoubleDataset.class: CompoundDoubleDataset.class);
    1011                        } else if (!keepLazy) {
    1012                                final int is = getElementsPerItem();
    1013                                try {
    1014                                        d = DatasetUtils.cast(is == 1 ? DoubleDataset.class: CompoundDoubleDataset.class, d.getSlice());
    1015                                } catch (DatasetException e) {
    1016                                        logger.error("Could not get data from lazy dataset", e);
    1017                                        return null;
    1018                                }
    1019                        }
    1020                } else {
    1021                        final int is = getElementsPerItem();
    1022                        if (is == 1) {
    1023                                d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
    1024                        } else {
    1025                                try {
    1026                                        d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob);
    1027                                } catch (IllegalArgumentException e) { // if only single value supplied try again
    1028                                        d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
    1029                                }
    1030                        }
    1031                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
    1032                                d.setShape(shape.clone());
    1033                        }
    1034                }
    1035                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
    1036                d.setShape(s.get(0));
    1037
    1038                return d;
    1039        }
    1040
    1041        @Override
    1042        public void setErrors(Serializable errors) {
    1043                if (shape == null) {
    1044                        throw new IllegalArgumentException("Cannot set errors for null dataset");
    1045                }
    1046                if (errors == null) {
    1047                        clearMetadata(ErrorMetadata.class);
    1048                        return;
    1049                }
    1050                if (errors == this) {
    1051                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
    1052                        return;
    1053                }
    1054
    1055                ILazyDataset errorData = createFromSerializable(errors, true);
    1056
    1057                ErrorMetadata emd = getErrorMetadata();
    1058                if (emd == null) {
    1059                        try {
    1060                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1061                                setMetadata(emd);
    1062                        } catch (MetadataException me) {
    1063                                logger.error("Could not create metadata", me);
    1064                        }
    1065                }
    1066                emd.setError(errorData);
    1067        }
    1068
    1069        protected ErrorMetadata getErrorMetadata() {
    1070                try {
    1071                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
    1072                        if (el != null && !el.isEmpty()) {
    1073                                 return el.get(0);
    1074                        }
    1075                } catch (Exception e) {
    1076                }
    1077                return null;
    1078        }
    1079
    1080        @Override
    1081        public ILazyDataset getErrors() {
    1082                ErrorMetadata emd = getErrorMetadata();
    1083                return emd == null ? null : emd.getError();
    1084        }
    1085
    1086        @Override
    1087        public boolean hasErrors() {
    1088                return LazyDatasetBase.this.getErrors() != null;
    1089        }
    1090
    1091        /**
    1092         * Check permutation axes
    1093         * @param shape to use
    1094         * @param axes if zero length then axes order reversed
    1095         * @return cleaned up copy of axes or null if trivial
    1096         */
    1097        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
    1098                int rank = shape == null ? 0 : shape.length;
    1099
    1100                if (axes == null || axes.length == 0) {
    1101                        axes = new int[rank];
    1102                        for (int i = 0; i < rank; i++) {
    1103                                axes[i] = rank - 1 - i;
    1104                        }
    1105                } else {
    1106                        axes = axes.clone();
    1107                }
    1108
    1109                if (axes.length != rank) {
    1110                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
    1111                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
    1112                }
    1113
    1114                // check all permutation values are within bounds
    1115                for (int i = 0; i < rank; i++) {
    1116                        axes[i] = ShapeUtils.checkAxis(rank, axes[i]);
    1117                }
    1118
    1119                // check for a valid permutation (is this an unnecessary restriction?)
    1120                int[] perm = axes.clone();
    1121                Arrays.sort(perm);
    1122
    1123                for (int i = 0; i < rank; i++) {
    1124                        if (perm[i] != i) {
    1125                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
    1126                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");
    1127                        }
    1128                }
    1129
    1130                if (Arrays.equals(axes, perm)) {
    1131                        return null; // signal identity or trivial permutation
    1132                }
    1133
    1134                return axes;
    1135        }
    1136}