    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.annotation.Annotation;
    017import java.lang.reflect.Array;
    018import java.lang.reflect.Field;
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.Collection;
    022import java.util.concurrent.ConcurrentMap;
    023import java.util.concurrent.ConcurrentHashMap;
    024import java.util.List;
    025import java.util.Map;
    026
    027import org.eclipse.january.DatasetException;
    028import org.eclipse.january.MetadataException;
    029import org.eclipse.january.metadata.Dirtiable;
    030import org.eclipse.january.metadata.ErrorMetadata;
    031import org.eclipse.january.metadata.IMetadata;
    032import org.eclipse.january.metadata.MetadataFactory;
    033import org.eclipse.january.metadata.MetadataType;
    034import org.eclipse.january.metadata.Reshapeable;
    035import org.eclipse.january.metadata.Sliceable;
    036import org.eclipse.january.metadata.Transposable;
    037import org.slf4j.Logger;
    038import org.slf4j.LoggerFactory;
    039
    040/**
    041 * Common base for both lazy and normal dataset implementations
    042 */
    043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
    044
    045        private static final long serialVersionUID = 767926846438976050L;
    046
    047        protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
    048
    049        protected static boolean catchExceptions;
    050
    051        static {
    052                /**
    053                 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE
    054                 */
    055                catchExceptions = Boolean.getBoolean("run.in.eclipse");
    056        }
    057
    058        protected String name = "";
    059
    060        /**
    061         * The shape or dimensions of the dataset
    062         */
    063        protected int[] shape;
    064
    065        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
    066
    067        /**
    068         * @return type of dataset item
    069         */
    070        abstract public int getDType();
    071
    072        @Override
    073        public Class<?> getElementClass() {
    074                return DTypeUtils.getElementClass(getDType());
    075        }
    076
    077        @Override
    078        public LazyDatasetBase clone() {
    079                return null;
    080        }
    081
    082        @Override
    083        public boolean equals(Object obj) {
    084                if (this == obj) {
    085                        return true;
    086                }
    087                if (obj == null) {
    088                        return false;
    089                }
    090                if (!getClass().equals(obj.getClass())) {
    091                        return false;
    092                }
    093
    094                LazyDatasetBase other = (LazyDatasetBase) obj;
    095                if (getDType() != other.getDType()) {
    096                        return false;
    097                }
    098                if (getElementsPerItem() != other.getElementsPerItem()) {
    099                        return false;
    100                }
    101                if (!Arrays.equals(shape, other.shape)) {
    102                        return false;
    103                }
    104                return true;
    105        }
    106
    107        @Override
    108        public int hashCode() {
    109                int hash = getDType() * 17 + getElementsPerItem();
    110                int rank = shape.length;
    111                for (int i = 0; i < rank; i++) {
    112                        hash = hash*17 + shape[i];
    113                }
    114                return hash;
    115        }
    116
    117        @Override
    118        public String getName() {
    119                return name;
    120        }
    121
    122        @Override
    123        public void setName(String name) {
    124                this.name = name;
    125        }
    126
    127        @Override
    128        public int[] getShape() {
    129                return shape.clone();
    130        }
    131
    132        @Override
    133        public int getRank() {
    134                return shape.length;
    135        }
    136
    137        /**
    138         * Find first sub-interface of (or class that directly implements) MetadataType
    139         * @param clazz
    140         * @return sub-interface
    141         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
    142         */
    143        @SuppressWarnings("unchecked")
    144        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
    145                if (clazz.equals(MetadataType.class)) {
    146                        throw new IllegalArgumentException("Cannot accept MetadataType");
    147                }
    148
    149                if (clazz.isInterface()) {
    150                        return clazz;
    151                }
    152
    153                if (clazz.isAnonymousClass()) { // special case
    154                        Class<?> s = clazz.getSuperclass();
    155                        if (!s.equals(Object.class)) {
    156                                // only use super class if it is not an anonymous class of an interface
    157                                clazz = (Class<? extends MetadataType>) s;
    158                        }
    159                }
    160
    161                for (Class<?> c : clazz.getInterfaces()) {
    162                        if (c.equals(MetadataType.class)) {
    163                                if (clazz.isAnonymousClass()) {
    164                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
    165                                }
    166                                return clazz;
    167                        }
    168                        if (MetadataType.class.isAssignableFrom(c)) {
    169                                return (Class<? extends MetadataType>) c;
    170                        }
    171                }
    172
    173                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
    174                if (c != null) {
    175                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
    176                }
    177
    178                logger.error("Somehow the search for metadata type interface ended in a bad place");
    179                assert false; // should not be able to get here!!!
    180                return null;
    181        }
    182
    183        @Override
    184        public void setMetadata(MetadataType metadata) {
    185                addMetadata(metadata, true);
    186        }
    187
    188        @Override
    189        public void addMetadata(MetadataType metadata) {
    190                addMetadata(metadata, false);
    191        }
    192
    193        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
    194                if (metadata == null)
    195                        return;
    196
    197                if (this.metadata == null) {
    198                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    199                }
    200
    201                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
    202                if (!this.metadata.containsKey(clazz)) {
    203                        this.metadata.put(clazz, new ArrayList<MetadataType>());
    204                } else if (clear) {
    205                        this.metadata.get(clazz).clear();
    206                }
    207                this.metadata.get(clazz).add(metadata);
    208
    209                // add for special case of sub-interfaces of IMetadata
    210                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
    211                        clazz = IMetadata.class;
    212                        if (!this.metadata.containsKey(clazz)) {
    213                                this.metadata.put(clazz, new ArrayList<MetadataType>());
    214                        } else if (clear) {
    215                                this.metadata.get(clazz).clear();
    216                        }
    217                        this.metadata.get(clazz).add(metadata);
    218                }
    219        }
    220
    221        @Override
    222        @Deprecated
    223        public synchronized IMetadata getMetadata() {
    224                return getFirstMetadata(IMetadata.class);
    225        }
    226
    227        @SuppressWarnings("unchecked")
    228        @Override
    229        public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException {
    230                if (metadata == null)
    231                        return null;
    232
    233                if (clazz == null) {
    234                        List<S> all = new ArrayList<S>();
    235                        for (Class<? extends MetadataType> c : metadata.keySet()) {
    236                                all.addAll((Collection<S>) metadata.get(c));
    237                        }
    238                        return all;
    239                }
    240
    241                return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz));
    242        }
    243
    244        @Override
    245        public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) {
    246                try {
    247                        List<S> ml = getMetadata(clazz);
    248                        if (ml == null) return null;
    249                        for (S t : ml) {
    250                                if (clazz.isInstance(t)) return t;
    251                        }
    252                } catch (Exception e) {
    253                        logger.error("Get metadata failed!",e);
    254                }
    255
    256                return null;
    257        }
    258
    259        @Override
    260        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
    261                if (metadata == null)
    262                        return;
    263
    264                if (clazz == null) {
    265                        metadata.clear();
    266                        return;
    267                }
    268
    269                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
    270                if( list != null) {
    271                        list.clear();
    272                }
    273        }
    274
    275        /**
    276         * @since 2.0
    277         */
    278        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
    279                return copyMetadata(metadata);
    280        }
    281
    282        /**
    283         * @since 2.0
    284         */
    285        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
    286                if (metadata == null)
    287                        return null;
    288
    289                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    290
    291                for (Class<? extends MetadataType> c : metadata.keySet()) {
    292                        List<MetadataType> l = metadata.get(c);
    293                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
    294                        map.put(c, nl);
    295                        for (MetadataType m : l) {
    296                                nl.add(m.clone());
    297                        }
    298                }
    299                return map;
    300        }
    301
    302        interface MetadatasetAnnotationOperation {
    303                /**
    304                 * Process value of given field
    305                 * <p>
    306                 * When the field is not a container then the returned value
    307                 * may replace the old value
    308                 * @param f given field
    309                 * @param o value of field
    310                 * @return transformed field
    311                 */
    312                Object processField(Field f, Object o);
    313
    314                /**
    315                 * @return annotated class
    316                 */
    317                Class<? extends Annotation> getAnnClass();
    318
    319                /**
    320                 * @param axis
    321                 * @return number of dimensions to insert or remove
    322                 */
    323                int change(int axis);
    324
    325                /**
    326                 *
    327                 * @return rank or -1 to match
    328                 */
    329                int getNewRank();
    330
    331                /**
    332                 * Run on given lazy dataset
    333                 * @param lz
    334                 * @return
    335                 */
    336                ILazyDataset run(ILazyDataset lz);
    337        }
    338
    339        class MdsSlice implements MetadatasetAnnotationOperation {
    340                private boolean asView;
    341                private int[] start;
    342                private int[] stop;
    343                private int[] step;
    344                private int[] oShape;
    345                private long oSize;
    346
    347                public MdsSlice(boolean asView, final int[] start, final int[] stop, final int[] step, final int[] oShape) {
    348                        this.asView = asView;
    349                        this.start = start;
    350                        this.stop = stop;
    351                        this.step = step;
    352                        this.oShape = oShape;
    353                        oSize = ShapeUtils.calcLongSize(oShape);
    354                }
    355
    356                @Override
    357                public Object processField(Field field, Object o) {
    358                        return o;
    359                }
    360
    361                @Override
    362                public Class<? extends Annotation> getAnnClass() {
    363                        return Sliceable.class;
    364                }
    365
    366                @Override
    367                public int change(int axis) {
    368                        return 0;
    369                }
    370
    371                @Override
    372                public int getNewRank() {
    373                        return -1;
    374                }
    375
    376                @Override
    377                public ILazyDataset run(ILazyDataset lz) {
    378                        int rank = lz.getRank();
    379                        if (start.length != rank) {
    380                                throw new IllegalArgumentException("Slice dimensions do not match dataset!");
    381                        }
    382
    383                        int[] shape = lz.getShape();
    384                        int[] stt;
    385                        int[] stp;
    386                        int[] ste;
    387                        if (lz.getSize() == oSize) {
    388                                stt = start;
    389                                stp = stop;
    390                                ste = step;
    391                        } else {
    392                                stt = start.clone();
    393                                stp = stop.clone();
    394                                ste = step.clone();
    395                                for (int i = 0; i < rank; i++) {
    396                                        if (shape[i] >= oShape[i]) continue;
    397                                        if (shape[i] == 1) {
    398                                                stt[i] = 0;
    399                                                stp[i] = 1;
    400                                                ste[1] = 1;
    401                                        } else {
    402                                                throw new IllegalArgumentException("Sliceable dataset has invalid size!");
    403                                        }
    404                                }
    405                        }
    406
    407                        if (asView || (lz instanceof IDataset))
    408                                return lz.getSliceView(stt, stp, ste);
    409                        try {
    410                                return lz.getSlice(stt, stp, ste);
    411                        } catch (DatasetException e) {
    412                                logger.error("Could not slice dataset in metadata", e);
    413                                return null;
    414                        }
    415                }
    416        }
    417
    418        class MdsReshape implements MetadatasetAnnotationOperation {
    419                private boolean matchRank;
    420                private int[] oldShape;
    421                private int[] newShape;
    422                boolean onesOnly;
    423                int[] differences;
    424
    425                /*
    426                 * if only ones then record differences (insertions and deletions)
    427                 *
    428                 * if shape changing, find broadcasted dimensions and disallow
    429                 * merging that include those dimensions
    430                 */
    431                public MdsReshape(final int[] oldShape, final int[] newShape) {
    432                        this.oldShape = oldShape;
    433                        this.newShape = newShape;
    434                        differences = null;
    435                }
    436
    437                @Override
    438                public Object processField(Field field, Object o) {
    439                        Annotation a = field.getAnnotation(Reshapeable.class);
    440                        if (a != null) { // cannot be null
    441                                matchRank = ((Reshapeable) a).matchRank();
    442                        }
    443                        return o;
    444                }
    445
    446                @Override
    447                public Class<? extends Annotation> getAnnClass() {
    448                        return Reshapeable.class;
    449                }
    450
    451                @Override
    452                public int change(int axis) {
    453                        if (matchRank) {
    454                                if (differences == null)
    455                                        init();
    456
    457                                if (onesOnly) {
    458                                        return differences[axis];
    459                                }
    460                                throw new UnsupportedOperationException("TODO support other shape operations");
    461                        }
    462                        return 0;
    463                }
    464
    465                @Override
    466                public int getNewRank() {
    467                        return matchRank ? newShape.length : -1;
    468                }
    469
    470                private void init() {
    471                        int or = oldShape.length - 1;
    472                        int nr = newShape.length - 1;
    473                        if (or < 0 || nr < 0) { // zero-rank shapes
    474                                onesOnly = true;
    475                                differences = new int[1];
    476                                differences[0] = or < 0 ? nr + 1 : or + 1;
    477                                return;
    478                        }
    479                        int ob = 0;
    480                        int nb = 0;
    481                        onesOnly = true;
    482                        do {
    483                                while (oldShape[ob] == 1 && ob < or) {
    484                                        ob++; // next non-unit dimension
    485                                }
    486                                while (newShape[nb] == 1 && nb < nr) {
    487                                        nb++;
    488                                }
    489                                if (oldShape[ob++] != newShape[nb++]) {
    490                                        onesOnly = false;
    491                                        break;
    492                                }
    493                        } while (ob <= or && nb <= nr);
    494
    495                        ob = 0;
    496                        nb = 0;
    497                        differences = new int[or + 2];
    498                        if (onesOnly) {
    499                                // work out unit dimensions removed from or add to old
    500                                int j = 0;
    501                                do {
    502                                        if (oldShape[ob] != 1 && newShape[nb] != 1) {
    503                                                ob++;
    504                                                nb++;
    505                                        } else {
    506                                                while (oldShape[ob] == 1 && ob < or) {
    507                                                        ob++;
    508                                                        differences[j]--;
    509                                                }
    510                                                while (newShape[nb] == 1 && nb < nr) {
    511                                                        nb++;
    512                                                        differences[j]++;
    513                                                }
    514                                        }
    515                                        j++;
    516                                } while (ob <= or && nb <= nr && j <= or);
    517                                while (ob <= or && oldShape[ob] == 1) {
    518                                        ob++;
    519                                        differences[j]--;
    520                                }
    521                                while (nb <= nr && newShape[nb] == 1) {
    522                                        nb++;
    523                                        differences[j]++;
    524                                }
    525                        } else {
    526                                if (matchRank) {
    527                                        logger.error("Combining dimensions is currently not supported");
    528                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
    529                                }
    530                                // work out mapping: contiguous dimensions can be grouped or split
    531                                while (ob <= or && nb <= nr) {
    532                                        int ol = oldShape[ob];
    533                                        while (ol == 1 && ol <= or) {
    534                                                ob++;
    535                                                ol = oldShape[ob];
    536                                        }
    537                                        int oe = ob + 1;
    538                                        int nl = newShape[nb];
    539                                        while (nl == 1 && nl <= nr) {
    540                                                nb++;
    541                                                nl = newShape[nb];
    542                                        }
    543                                        int ne = nb + 1;
    544                                        if (ol < nl) {
    545                                                differences[ob] = 1;
    546                                                do { // case where new shape combines several dimensions into one dimension
    547                                                        if (oe == (or + 1)) {
    548                                                                break;
    549                                                        }
    550                                                        differences[oe] = 1;
    551                                                        ol *= oldShape[oe++];
    552                                                } while (ol < nl);
    553                                                differences[oe - 1] = oe - ob; // signal end with difference
    554                                                if (nl != ol) {
    555                                                        logger.error("Single dimension is incompatible with subshape");
    556                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    557                                                }
    558                                        } else if (ol > nl) {
    559                                                do { // case where new shape spreads single dimension over several dimensions
    560                                                        if (ne == (nr + 1)) {
    561                                                                break;
    562                                                        }
    563                                                        nl *= newShape[ne++];
    564                                                } while (nl < ol);
    565                                                if (nl != ol) {
    566                                                        logger.error("Subshape is incompatible with single dimension");
    567                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    568                                                }
    569
    570                                        }
    571
    572                                        ob = oe;
    573                                        nb = ne;
    574                                }
    575
    576                        }
    577                }
    578
    579                @Override
    580                public ILazyDataset run(ILazyDataset lz) {
    581                        if (differences == null)
    582                                init();
    583
    584                        int[] lshape = lz.getShape();
    585                        if (Arrays.equals(newShape, lshape)) {
    586                                return lz;
    587                        }
    588                        int or = lz.getRank();
    589                        int nr = newShape.length;
    590                        int[] nshape = new int[nr];
    591                        Arrays.fill(nshape, 1);
    592                        if (onesOnly) {
    593                                // ignore omit removed dimensions
    594                                for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) {
    595                                        int c = differences[i];
    596                                        if (c == 0) {
    597                                                nshape[di++] = lshape[si++];
    598                                        } else if (c > 0) {
    599                                                while (c-- > 0 && di < nr) {
    600                                                        di++;
    601                                                }
    602                                        } else if (c < 0) {
    603                                                si -= c; // remove dimensions by skipping forward in source array
    604                                        }
    605                                }
    606                        } else {
    607                                boolean[] broadcast = new boolean[or];
    608                                for (int ob = 0; ob < or; ob++) {
    609                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
    610                                }
    611                                int osize = lz.getSize();
    612
    613                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
    614                                int ob = 0;
    615                                int nsize = 1;
    616                                for (int i = 0; i < nr; i++) {
    617                                        if (ob < or && broadcast[ob]) {
    618                                                if (differences[ob] != 0) {
    619                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
    620                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
    621                                                }
    622                                        } else {
    623                                                nshape[i] = nsize < osize ? newShape[i] : 1;
    624                                        }
    625                                        nsize *= nshape[i];
    626                                        ob++;
    627                                }
    628                        }
    629
    630                        ILazyDataset nlz = lz.getSliceView();
    631                        if (lz instanceof Dataset) {
    632                                nlz = ((Dataset) lz).reshape(nshape);
    633                        } else {
    634                                nlz = lz.getSliceView();
    635                                nlz.setShape(nshape);
    636                        }
    637                        return nlz;
    638                }
    639        }
    640
    641        class MdsTranspose implements MetadatasetAnnotationOperation {
    642                int[] map;
    643
    644                public MdsTranspose(final int[] axesMap) {
    645                        map = axesMap;
    646                }
    647
    648                @SuppressWarnings({ "rawtypes", "unchecked" })
    649                @Override
    650                public Object processField(Field f, Object o) {
    651                        // reorder arrays and lists according the axes map
    652                        if (o.getClass().isArray()) {
    653                                int l = Array.getLength(o);
    654                                if (l == map.length) {
    655                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    656                                        for (int i = 0; i < l; i++) {
    657                                                Array.set(narray, i, Array.get(o, map[i]));
    658                                        }
    659                                        for (int i = 0; i < l; i++) {
    660                                                Array.set(o, i, Array.get(narray, i));
    661                                        }
    662                                }
    663                        } else if (o instanceof List<?>) {
    664                                List list = (List) o;
    665                                int l = list.size();
    666                                if (l == map.length) {
    667                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    668                                        for (int i = 0; i < l; i++) {
    669                                                Array.set(narray, i, list.get(map[i]));
    670                                        }
    671                                        list.clear();
    672                                        for (int i = 0; i < l; i++) {
    673                                                list.add(Array.get(narray, i));
    674                                        }
    675                                }
    676                        }
    677                        return o;
    678                }
    679
    680                @Override
    681                public Class<? extends Annotation> getAnnClass() {
    682                        return Transposable.class;
    683                }
    684
    685                @Override
    686                public int change(int axis) {
    687                        return 0;
    688                }
    689
    690                @Override
    691                public int getNewRank() {
    692                        return -1;
    693                }
    694
    695                @Override
    696                public ILazyDataset run(ILazyDataset lz) {
    697                        return lz.getTransposedView(map);
    698                }
    699        }
    700
    701        class MdsDirty implements MetadatasetAnnotationOperation {
    702
    703                @Override
    704                public Object processField(Field f, Object o) {
    705                        // throw exception if not boolean???
    706                        Class<?> t = f.getType();
    707                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
    708                                if (o.equals(false)) {
    709                                        o = true;
    710                                }
    711                        }
    712                        return o;
    713                }
    714
    715                @Override
    716                public Class<? extends Annotation> getAnnClass() {
    717                        return Dirtiable.class;
    718                }
    719
    720                @Override
    721                public int change(int axis) {
    722                        return 0;
    723                }
    724
    725                @Override
    726                public int getNewRank() {
    727                        return -1;
    728                }
    729
    730                @Override
    731                public ILazyDataset run(ILazyDataset lz) {
    732                        return lz;
    733                }
    734        }
    735
    736        /**
    737         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
    738         * dataset after cloning the metadata
    739         * @param asView if true then just a view
    740         * @param slice
    741         */
    742        protected void sliceMetadata(boolean asView, final SliceND slice) {
    743                processAnnotatedMetadata(new MdsSlice(asView, slice.getStart(), slice.getStop(), slice.getStep(), slice.getSourceShape()), true);
    744        }
    745
    746        /**
    747         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
    748         * or setting the shape
    749         *
    750         * @param newShape
    751         */
    752        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
    753                processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true);
    754        }
    755
    756        /**
    757         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
    758         * dataset after cloning the metadata
    759         * @param axesMap
    760         */
    761        protected void transposeMetadata(final int[] axesMap) {
    762                processAnnotatedMetadata(new MdsTranspose(axesMap), true);
    763        }
    764
    765        /**
    766         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
    767         * @since 2.0
    768         */
    769        protected void dirtyMetadata() {
    770                processAnnotatedMetadata(new MdsDirty(), true);
    771        }
    772
    773        @SuppressWarnings("unchecked")
    774        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) {
    775                if (metadata == null)
    776                        return;
    777
    778                for (Class<? extends MetadataType> c : metadata.keySet()) {
    779                        for (MetadataType m : metadata.get(c)) {
    780                                if (m == null)
    781                                        continue;
    782
    783                                Class<? extends MetadataType> mc = m.getClass();
    784                                do { // iterate over super-classes
    785                                        processClass(op, m, mc, throwException);
    786                                        Class<?> sclazz = mc.getSuperclass();
    787                                        if (!MetadataType.class.isAssignableFrom(sclazz))
    788                                                break;
    789                                        mc = (Class<? extends MetadataType>) sclazz;
    790                                } while (true);
    791                        }
    792                }
    793        }
    794
    795        @SuppressWarnings({ "unchecked", "rawtypes" })
    796        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) {
    797                for (Field f : mc.getDeclaredFields()) {
    798                        if (!f.isAnnotationPresent(op.getAnnClass()))
    799                                continue;
    800
    801                        try {
    802                                f.setAccessible(true);
    803                                Object o = f.get(m);
    804                                if (o == null)
    805                                        continue;
    806
    807                                Object no = op.processField(f, o);
    808                                if (no != o) {
    809                                        f.set(m, no);
    810                                        continue;
    811                                }
    812                                Object r = null;
    813                                if (o instanceof ILazyDataset) {
    814                                        try {
    815                                                f.set(m, op.run((ILazyDataset) o));
    816                                        } catch (Exception e) {
    817                                                logger.error("Problem processing " + o, e);
    818                                                if (!catchExceptions)
    819                                                        throw e;
    820                                        }
    821                                } else if (o.getClass().isArray()) {
    822                                        int l = Array.getLength(o);
    823                                        if (l <= 0)
    824                                                continue;
    825
    826                                        for (int i = 0; r == null && i < l; i++) {
    827                                                r = Array.get(o, i);
    828                                        }
    829                                        int n = op.getNewRank();
    830                                        if (r == null) {
    831                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    832                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
    833                                                }
    834                                                continue;
    835                                        }
    836                                        if (n < 0)
    837                                                n = l;
    838                                        Object narray = Array.newInstance(r.getClass(), n);
    839                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
    840                                                int c = op.change(i);
    841                                                if (c == 0) {
    842                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
    843                                                } else if (c > 0) {
    844                                                        di += c; // add nulls by skipping forward in destination array
    845                                                } else if (c < 0) {
    846                                                        si -= c; // remove dimensions by skipping forward in source array
    847                                                }
    848                                        }
    849                                        if (n == l) {
    850                                                for (int i = 0; i < l; i++) {
    851                                                        Array.set(o, i, Array.get(narray, i));
    852                                                }
    853                                        } else {
    854                                                f.set(m, narray);
    855                                        }
    856                                } else if (o instanceof List<?>) {
    857                                        List list = (List) o;
    858                                        int l = list.size();
    859                                        if (l <= 0)
    860                                                continue;
    861
    862                                        for (int i = 0; r == null && i < l; i++) {
    863                                                r = list.get(i);
    864                                        }
    865                                        int n = op.getNewRank();
    866                                        if (r == null) {
    867                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    868                                                        list.clear();
    869                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
    870                                                                list.add(null);
    871                                                        }
    872                                                }
    873                                                continue;
    874                                        }
    875
    876                                        if (n < 0)
    877                                                n = l;
    878                                        Object narray = Array.newInstance(r.getClass(), n);
    879                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
    880                                                int c = op.change(i);
    881                                                if (c == 0) {
    882                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
    883                                                } else if (c > 0) {
    884                                                        di += c; // add nulls by skipping forward in destination array
    885                                                } else if (c < 0) {
    886                                                        si -= c; // remove dimensions by skipping forward in source array
    887                                                }
    888                                        }
    889                                        list.clear();
    890                                        for (int i = 0; i < n; i++) {
    891                                                list.add(Array.get(narray, i));
    892                                        }
    893                                } else if (o instanceof Map<?,?>) {
    894                                        Map map = (Map) o;
    895                                        for (Object k : map.keySet()) {
    896                                                map.put(k, processObject(op, map.get(k)));
    897                                        }
    898                                }
    899                        } catch (Exception e) {
    900                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
    901                                if (throwException)
    902                                        throw new RuntimeException(e);
    903                        }
    904                }
    905        }
    906
    907        @SuppressWarnings({ "unchecked", "rawtypes" })
    908        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
    909                if (o == null)
    910                        return o;
    911
    912                if (o instanceof ILazyDataset) {
    913                        try {
    914                                return op.run((ILazyDataset) o);
    915                        } catch (Exception e) {
    916                                logger.error("Problem processing " + o, e);
    917                                if (!catchExceptions)
    918                                        throw e;
    919                        }
    920                } else if (o.getClass().isArray()) {
    921                        int l = Array.getLength(o);
    922                        for (int i = 0; i < l; i++) {
    923                                Array.set(o, i, processObject(op, Array.get(o, i)));
    924                        }
    925                } else if (o instanceof List<?>) {
    926                        List list = (List) o;
    927                        for (int i = 0, imax = list.size(); i < imax; i++) {
    928                                list.set(i, processObject(op, list.get(i)));
    929                        }
    930                } else if (o instanceof Map<?,?>) {
    931                        Map map = (Map) o;
    932                        for (Object k : map.keySet()) {
    933                                map.put(k, processObject(op, map.get(k)));
    934                        }
    935                }
    936                return o;
    937        }
    938
    939        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
    940                for (Class<? extends MetadataType> mc : oldMetadata.keySet()) {
    941                        metadata.put(mc, oldMetadata.get(mc));
    942                }
    943        }
    944
    945        @SuppressWarnings("deprecation")
    946        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
    947                ILazyDataset d = null;
    948                if (blob instanceof ILazyDataset) {
    949                        d = (ILazyDataset) blob;
    950                        if (d instanceof IDataset) {
    951                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
    952                                int is = ed.getElementsPerItem();
    953                                if (is != 1 && is != getElementsPerItem()) {
    954                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
    955                                }
    956                                d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
    957                        } else if (!keepLazy) {
    958                                final int is = getElementsPerItem();
    959                                try {
    960                                        d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
    961                                } catch (DatasetException e) {
    962                                        logger.error("Could not get data from lazy dataset", e);
    963                                        return null;
    964                                }
    965                        }
    966                } else {
    967                        final int is = getElementsPerItem();
    968                        if (is == 1) {
    969                                d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob);
    970                        } else {
    971                                try {
    972                                        d = DatasetFactory.createFromObject(is, Dataset.ARRAYFLOAT64, blob);
    973                                } catch (IllegalArgumentException e) { // if only single value supplied try again
    974                                        d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob);
    975                                }
    976                        }
    977                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
    978                                d.setShape(shape.clone());
    979                        }
    980                }
    981                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
    982                d.setShape(s.get(0));
    983
    984                return d;
    985        }
    986
    987        @Override
    988        public void setErrors(Serializable errors) {
    989                if (shape == null) {
    990                        throw new IllegalArgumentException("Cannot set errors for null dataset");
    991                }
    992                if (errors == null) {
    993                        clearMetadata(ErrorMetadata.class);
    994                        return;
    995                }
    996                if (errors == this) {
    997                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
    998                        return;
    999                }
    1000
    1001                ILazyDataset errorData = createFromSerializable(errors, true);
    1002
    1003                ErrorMetadata emd = getErrorMetadata();
    1004                if (emd == null) {
    1005                        try {
    1006                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1007                                setMetadata(emd);
    1008                        } catch (MetadataException me) {
    1009                                logger.error("Could not create metadata", me);
    1010                        }
    1011                }
    1012                emd.setError(errorData);
    1013        }
    1014
    1015        protected ErrorMetadata getErrorMetadata() {
    1016                try {
    1017                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
    1018                        if (el != null && !el.isEmpty()) {
    1019                                 return el.get(0);
    1020                        }
    1021                } catch (Exception e) {
    1022                }
    1023                return null;
    1024        }
    1025
    1026        @Override
    1027        public ILazyDataset getErrors() {
    1028                ErrorMetadata emd = getErrorMetadata();
    1029                return emd == null ? null : emd.getError();
    1030        }
    1031
    1032        @Override
    1033        public boolean hasErrors() {
    1034                return LazyDatasetBase.this.getErrors() != null;
    1035        }
    1036
    1037        /**
    1038         * Check permutation axes
    1039         * @param shape
    1040         * @param axes
    1041         * @return cleaned up axes or null if trivial
    1042         */
    1043        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
    1044                int rank = shape.length;
    1045
    1046                if (axes == null || axes.length == 0) {
    1047                        axes = new int[rank];
    1048                        for (int i = 0; i < rank; i++) {
    1049                                axes[i] = rank - 1 - i;
    1050                        }
    1051                }
    1052
    1053                if (axes.length != rank) {
    1054                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
    1055                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
    1056                }
    1057
    1058                // check all permutation values are within bounds
    1059                for (int d : axes) {
    1060                        if (d < 0 || d >= rank) {
    1061                                logger.error("axis permutation contains element {} outside rank of dataset", d);
    1062                                throw new IllegalArgumentException("axis permutation contains element outside rank of dataset");
    1063                        }
    1064                }
    1065
    1066                // check for a valid permutation (is this an unnecessary restriction?)
    1067                int[] perm = axes.clone();
    1068                Arrays.sort(perm);
    1069
    1070                for (int i = 0; i < rank; i++) {
    1071                        if (perm[i] != i) {
    1072                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
    1073                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");
    1074                        }
    1075                }
    1076
    1077                if (Arrays.equals(axes, perm))
    1078                        return null; // signal identity or trivial permutation
    1079
    1080                return axes;
    1081        }
    1082}