Python numpy.setdiff1d() 使用实例

The following are code examples for showing how to use . They are extracted from open source Python projects. You can vote up the examples you like or vote down the exmaples you don’t like. You can also save this page to your account.

Example 1

def confirmContent_NoOrder(self):
        '''
        Confirms that all information is the same except for the maxRange field.
        Order is ignored
        '''
        sf = StationFinder()
        stations = sf.findStation(unitCode = self.unitCode, distance = self.distance,
            climateParameters = self.climateParameters, sdate = self.sdate, edate = self.edate)
        test_data = numpy.array(stations._dumpMetaToList())
        ref_data = []
        with open(Test_StationFinder.rootFolder + self.refFile, 'r')as refFile:
            r = csv.reader(refFile)
            for line in r:
                ref_data.append(line)
        ref_data = numpy.array(ref_data)

        refFile.close()
        self.results =  list(numpy.setdiff1d(ref_data[:,Test_StationFinder.testColumns]
            ,test_data[:,Test_StationFinder.testColumns])) 

Example 2

def confirmContent(self):
        '''
        Confirms that all information is the same, ignoring record order
        '''
        dr = StationDataRequestor()
        wxData =  dr.getDailyWxObservations(climateStations =  self.climateStations,
            climateParameters = self.climateParameters
            ,sdate = self.sdate, edate = self.edate)
        wxData.export('temp.csv')
        infile = open('temp.csv','r')
        testData = infile.read()
        refDataFile = open(Test_StationDataRequestor_getDailyWxObs.rootFolder + self.refDataFile, 'r')
        refData = refDataFile.read()
        infile.close()
        refDataFile.close()
        os.remove('temp.csv')
        self.result =  list(numpy.setdiff1d(refData.split('/n'), testData.split('/n'))) 

Example 3

def confirmContent(self):
        '''
        Confirms that all information is the same, ignoring record order
        '''
        dr = StationDataRequestor()
        wxData =  dr.getMonthlyWxSummaryByYear(climateStations =  self.climateStations,
            climateParameters = self.climateParameters, reduceCodes = self.reduceCodes
            ,sdate = self.sdate, edate = self.edate, maxMissing = self.maxMissing,
            includeNormals = self.includeNormals, includeNormalDepartures = self.includeNormalDepartures)
        wxData.export('temp.csv')
        infile = open('temp.csv','r')
        testData = infile.read()
        refDataFile = open(Test_StationDataRequestor_getMonthlyWxSummaryByYear.rootFolder + self.refDataFile, 'r')
        refData = refDataFile.read()
        infile.close()
        refDataFile.close()
        os.remove('temp.csv')
        self.result =  list(numpy.setdiff1d(refData.split('/n'), testData.split('/n'))) 

Example 4

def confirmContent(self):
        '''
        Confirms that all information is the same, ignoring record order
        '''
        dr = StationDataRequestor()
        wxData =  dr.getYearlyWxSummary(climateStations =  self.climateStations,
            climateParameters = self.climateParameters, reduceCodes = self.reduceCodes
            ,sdate = self.sdate, edate = self.edate, maxMissing = self.maxMissing,
            includeNormals = self.includeNormals, includeNormalDepartures = self.includeNormalDepartures)
        wxData.export('temp.csv')
        infile = open('temp.csv','r')
        testData = infile.read()
        refDataFile = open(Test_StationDataRequestor_getYearlyWxSummary.rootFolder + self.refDataFile, 'r')
        refData = refDataFile.read()
        infile.close()
        refDataFile.close()
        os.remove('temp.csv')
        self.result =  list(numpy.setdiff1d(refData.split('/n'), testData.split('/n'))) 

Example 5

def check_multiplication_dims(dims, N, M, vidx=False, without=False):
    dims = array(dims, ndmin=1)
    if len(dims) == 0:
        dims = arange(N)
    if without:
        dims = setdiff1d(range(N), dims)
    if not np.in1d(dims, arange(N)).all():
        raise ValueError('Invalid dimensions')
    P = len(dims)
    sidx = np.argsort(dims)
    sdims = dims[sidx]
    if vidx:
        if M > N:
            raise ValueError('More multiplicants than dimensions')
        if M != N and M != P:
            raise ValueError('Invalid number of multiplicants')
        if P == M:
            vidx = sidx
        else:
            vidx = sdims
        return sdims, vidx
    else:
        return sdims 

Example 6

def _annotate_epochs(self, threshes, epochs):
        """Get essential annotations for epochs given thresholds."""
        ch_type = _get_ch_type_from_picks(self.picks, epochs.info)[0]

        drop_log, bad_sensor_counts = self._vote_bad_epochs(epochs)

        interp_channels, fix_log = self._get_epochs_interpolation(
            epochs, drop_log=drop_log, ch_type=ch_type)

        (bad_epochs_idx, sorted_epoch_idx,
         n_epochs_drop) = self._get_bad_epochs(
             bad_sensor_counts, ch_type=ch_type)

        bad_epochs_idx = np.sort(bad_epochs_idx)
        good_epochs_idx = np.setdiff1d(np.arange(len(epochs)),
                                       bad_epochs_idx)

        return (drop_log, bad_sensor_counts, interp_channels, fix_log,
                bad_epochs_idx, good_epochs_idx) 

Example 7

def has_approx_support(m, m_hat, prob=0.01):
    """Returns 1 if model selection error is less than or equal to prob rate,
    0 else.

    NOTE: why does np.nonzero/np.flatnonzero create so much problems?
    """
    m_nz = np.flatnonzero(np.triu(m, 1))
    m_hat_nz = np.flatnonzero(np.triu(m_hat, 1))

    upper_diagonal_mask = np.flatnonzero(np.triu(np.ones(m.shape), 1))
    not_m_nz = np.setdiff1d(upper_diagonal_mask, m_nz)

    intersection = np.in1d(m_hat_nz, m_nz)  # true positives
    not_intersection = np.in1d(m_hat_nz, not_m_nz)  # false positives

    true_positive_rate = 0.0
    if len(m_nz):
        true_positive_rate = 1. * np.sum(intersection) / len(m_nz)
        true_negative_rate = 1. - true_positive_rate

    false_positive_rate = 0.0
    if len(not_m_nz):
        false_positive_rate = 1. * np.sum(not_intersection) / len(not_m_nz)

    return int(np.less_equal(true_negative_rate + false_positive_rate, prob)) 

Example 8

def transform(self, y):
        """Transform labels to normalized encoding.
        Parameters
        ----------
        y : array-like of shape [n_samples]
            Target values.
        Returns
        -------
        y : array-like of shape [n_samples]
        """
        y = column_or_1d(y, warn=True)

        classes = np.unique(y)
        if len(np.intersect1d(classes, self.classes_)) < len(classes):
            diff = np.setdiff1d(classes, self.classes_)
            self.classes_ = np.hstack((self.classes_, diff))
        return np.searchsorted(self.classes_, y)[0] 

Example 9

def initialize(self, coordinates):    #not run until after first selection!
        # set up mines
        # randomly place mines anywhere *except* first selected location AND surrounding cells
        # so that first selection is always a 0
        # weird, yes, but that's how the original minesweeper worked
        availableCells = range(self.totalCells)
        selected = coordinates[0]*self.dim2 + coordinates[1]
        offLimits = np.array([selected-self.dim2-1, selected-self.dim2, selected-self.dim2+1, selected-1, selected, selected+1, selected+self.dim2-1, selected+self.dim2, selected+self.dim2+1])    #out of bounds is ok
        availableCells = np.setdiff1d(availableCells, offLimits)
        self.nMines = np.minimum(self.nMines, len(availableCells))  #in case there are fewer remaining cells than mines to place
        minesFlattened = np.zeros([self.totalCells])
        minesFlattened[np.random.choice(availableCells, self.nMines, replace=False)] = 1
        self.mines = minesFlattened.reshape([self.dim1, self.dim2])
        # set up neighbors
        for i in range(self.dim1):
            for j in range(self.dim2):
                nNeighbors = 0
                for k in range(-1, 2):
                    if i + k >= 0 and i + k < self.dim1:
                        for l in range(-1, 2):
                            if j + l >= 0 and j + l < self.dim2 and (k != 0 or l != 0):
                                nNeighbors += self.mines[i + k, j + l]
                self.neighbors[i, j] = nNeighbors
        #done
        self.initialized = True 

Example 10

def find_neighbours(self, idx, features):
        """
        Finds the neighbours of the given point which are at a maximum distance
        of self.eps from it.

        :param  idx:        Index of the current point
        :param  features:   Dataset, array-like object of shape
                            (nb_samples, nb_features)
        :returns:           List containing the indexes of the neighbours
        """
        data = features[np.setdiff1d(np.arange(features.shape[0]), idx)]
        distances = self.get_distances(features[idx], data)
        same_cluster = [idx]
        for i, dist in enumerate(distances.tolist()[0]):
            real_index = i if i < idx else i + 1
            if dist <= self.eps:
                same_cluster.append(real_index)
        return same_cluster 

Example 11

def load_co2_data(prop=0.8):
    from sklearn.datasets import fetch_mldata
    from sklearn import cross_validation
    data = fetch_mldata('mauna-loa-atmospheric-co2').data
    X = data[:, [1]]
    y = data[:, 0]
    y = y[:, None]
    X = X.astype(np.float64)
    ntrain = y.shape[0]
    train_inds = npr.choice(range(ntrain), int(prop*ntrain), replace=False)
    valid_inds = np.setdiff1d(range(ntrain), train_inds)
    X_train, y_train = X[train_inds].copy(), y[train_inds].copy()
    X_valid, y_valid = X[valid_inds].copy(), y[valid_inds].copy()
    return X_train, y_train, X_valid, y_valid

############################ Training & Visualizing ############################ 

Example 12

def transform(self, y):
        """Transform labels to normalized encoding.

        Parameters
        ----------
        y : array-like of shape [n_samples]
            Target values.

        Returns
        -------
        y : array-like of shape [n_samples]
        """
        check_is_fitted(self, 'classes_')
        y = column_or_1d(y.ravel(), warn=True)
        classes = np.unique(y)
        if isinstance(classes[0], np.float64):
            classes = classes[np.isfinite(classes)]
        _check_numpy_unicode_bug(classes)
        if len(np.intersect1d(classes, self.classes_)) < len(classes):
            diff = np.setdiff1d(classes, self.classes_)
            print(self.classes_)
            raise ValueError("y contains new labels: %s" % str(diff))
        return np.searchsorted(self.classes_, y).reshape(-1, 1) 

Example 13

def inverse_transform(self, y):
        """Transform labels back to original encoding.

        Parameters
        ----------
        y : numpy array of shape [n_samples]
            Target values.

        Returns
        -------
        y : numpy array of shape [n_samples]
        """
        check_is_fitted(self, 'classes_')

        diff = np.setdiff1d(y, np.arange(len(self.classes_)))
        if diff:
            raise ValueError("y contains new labels: %s" % str(diff))
        y = np.asarray(y)
        return self.classes_[y] 

Example 14

def sortclusters(CoPaM, Mc, minGenesinClust = 11):
    Mcloc = np.array(Mc)
    [Np, K] = Mcloc.shape
    largerThanMax = np.max(Mcloc) + 1
    Cf = np.zeros(K, dtype=int) - 1

    for i in range(Np-1,-1,-1):
        C = np.argsort(Mcloc[i])[::-1]
        M = Mcloc[i,C]
        Cf[np.all([M >= minGenesinClust, Cf == 0], axis=0)] = C[np.all([M >= minGenesinClust, Cf == 0], axis=0)]
        if i > 0:
            Mcloc[i-1, Cf[Cf != 0]] = largerThanMax

    Cf[Cf==-1] = np.setdiff1d(np.arange(K), Cf)

    return np.array(CoPaM)[:, Cf]


# Clustering helping function for parallel loop 

Example 15

def test_fix_types():
    """Test fixing of channel types
    """
    for fname, change in ((hp_fif_fname, True), (test_fif_fname, False),
                          (ctf_fname, False)):
        raw = Raw(fname)
        mag_picks = pick_types(raw.info, meg='mag')
        other_picks = np.setdiff1d(np.arange(len(raw.ch_names)), mag_picks)
        # we don't actually have any files suffering from this problem, so
        # fake it
        if change:
            for ii in mag_picks:
                raw.info['chs'][ii]['coil_type'] = FIFF.FIFFV_COIL_VV_MAG_T2
        orig_types = np.array([ch['coil_type'] for ch in raw.info['chs']])
        raw.fix_mag_coil_types()
        new_types = np.array([ch['coil_type'] for ch in raw.info['chs']])
        if not change:
            assert_array_equal(orig_types, new_types)
        else:
            assert_array_equal(orig_types[other_picks], new_types[other_picks])
            assert_true((orig_types[mag_picks] != new_types[mag_picks]).all())
            assert_true((new_types[mag_picks] ==
                         FIFF.FIFFV_COIL_VV_MAG_T3).all()) 

Example 16

def get_finished_jobs(job_ids):
    """Get a list of finished job ids for the given list of jobs

    Keyword arguments:

    job_ids     -- list of lobs that shall be checked
    """
    data = get_qstat_as_df()
    finished_jobs = []

    if len(data) == 0:
        return job_ids

    ids_in_data = data[data["JOBID"].isin(job_ids)]
    finished_jobs = np.setdiff1d(job_ids, ids_in_data["JOBID"])

    return np.array(finished_jobs) 

Example 17

def fix_predictions(self, X, predictions, bias):
        idxs_users_missing, idxs_items_missing = self.indices_missing

        # Set average when neither the user nor the item exist
        g_avg = bias['globalAvg']
        common_indices = np.intersect1d(idxs_users_missing, idxs_items_missing)
        predictions[common_indices] = g_avg

        # Only users exist (return average + {dUser})
        if 'dUsers' in bias:
            missing_users = np.setdiff1d(idxs_users_missing, common_indices)
            if len(missing_users) > 0:
                user_idxs = X[missing_users, self.order[0]]
                predictions[missing_users] = g_avg + bias['dUsers'][user_idxs]

        # Only items exist (return average + {dItem})
        if 'dItems' in bias:
            missing_items = np.setdiff1d(idxs_items_missing, common_indices)
            if len(missing_items) > 0:
                item_idxs = X[missing_items, self.order[1]]
                predictions[missing_items] = g_avg + bias['dItems'][item_idxs]

        return predictions 

Example 18

def measure_background(image, Fibers, width=30, niter=3, order=3):
    t = []
    a,b = image.shape
    ygrid,xgrid = np.indices(image.shape)
    ygrid = 1. * ygrid.ravel() / a
    xgrid = 1. * xgrid.ravel() / b
    image = image.ravel()
    s = np.arange(a*b)
    for fiber in Fibers:
        t.append(fiber.D*fiber.yind + fiber.xind)
    t = np.hstack(t)
    t = np.array(t, dtype=int)
    ind = np.setdiff1d(s,t)
    mask = np.zeros((a*b))
    mask[ind] = 1.
    mask[ind] = 1.-is_outlier(image[ind])
    sel = np.where(mask==1.)[0]
    for i in xrange(niter):
        V = polyvander2d(xgrid[sel],ygrid[sel],[order,order])
        sol = np.linalg.lstsq(V, image[sel])[0]
        vals = np.dot(V,sol) - image[sel]
        sel = sel[~is_outlier(vals)]
    V = polyvander2d(xgrid,ygrid,[order,order])
    back = np.dot(V, sol).reshape(a,b)    
    return back 

Example 19

def _parallel_predict_log_proba(estimators, estimators_features, X, n_classes):
    """Private function used to compute log probabilities within a job."""
    n_samples = X.shape[0]
    log_proba = np.empty((n_samples, n_classes))
    log_proba.fill(-np.inf)
    all_classes = np.arange(n_classes, dtype=np.int)

    for estimator, features in zip(estimators, estimators_features):
        log_proba_estimator = estimator.predict_log_proba(X[:, features])

        if n_classes == len(estimator.classes_):
            log_proba = np.logaddexp(log_proba, log_proba_estimator)

        else:
            log_proba[:, estimator.classes_] = np.logaddexp(
                log_proba[:, estimator.classes_],
                log_proba_estimator[:, range(len(estimator.classes_))])

            missing = np.setdiff1d(all_classes, estimator.classes_)
            log_proba[:, missing] = np.logaddexp(log_proba[:, missing],
                                                 -np.inf)

    return log_proba 

Example 20

def transform(self, y):
        """Transform labels to normalized encoding.

        Parameters
        ----------
        y : array-like of shape [n_samples]
            Target values.

        Returns
        -------
        y : array-like of shape [n_samples]
        """
        check_is_fitted(self, 'classes_')
        y = column_or_1d(y, warn=True)

        classes = np.unique(y)
        _check_numpy_unicode_bug(classes)
        if len(np.intersect1d(classes, self.classes_)) < len(classes):
            diff = np.setdiff1d(classes, self.classes_)
            raise ValueError("y contains new labels: %s" % str(diff))
        return np.searchsorted(self.classes_, y) 

Example 21

def inverse_transform(self, y):
        """Transform labels back to original encoding.

        Parameters
        ----------
        y : numpy array of shape [n_samples]
            Target values.

        Returns
        -------
        y : numpy array of shape [n_samples]
        """
        check_is_fitted(self, 'classes_')

        diff = np.setdiff1d(y, np.arange(len(self.classes_)))
        if diff:
            raise ValueError("y contains new labels: %s" % str(diff))
        y = np.asarray(y)
        return self.classes_[y] 

Example 22

def _match_info(self):
        """
        Helper function to create match info
        """
        assert self.matches is not None, 'No matches yet!'
        self.matches = {
            'match_pairs' : self.matches,
            'treated' : np.unique(list(self.matches.keys())),
            'control' : np.unique(list(self.matches.values()))
        }
        self.matches['dropped'] = np.setdiff1d(list(range(self.nobs)), 
                                    np.append(self.matches['treated'], self.matches['control'])) 

Example 23

def confirmAsciiGrid(self):
        gr = GridRequestor()
        data =  gr.getGrids(sdate = self.sdate, edate = self.edate,
            unitCode = self.unitCode, distance = self.distance,
            climateParameters = self.climateParameters, duration = self.duration)
        testDataFile = data.export()[0]
        testFile = open(testDataFile,'r')
        testData = testFile.read()
        testFile.close()
        os.remove(testDataFile)
        os.remove(testDataFile[:-3] + 'prj')
        refDataFile = open(Test_GridRequestor.rootFolder + self.refDataFile,'r')
        refData = refDataFile.read()
        refDataFile.close()
        self.result =  list(numpy.setdiff1d(refData.split('/n'),testData.split('/n'))) 

Example 24

def prepare(data_valid):
    print(data_valid.shape)
    batch = data_valid.shape[0]
    N = data_valid.shape[1]
    data_invalid = np.random.randint(0,2,(batch,N),dtype=np.int8)
    print(data_valid.shape,data_invalid.shape)
    ai = data_invalid.view([('', data_invalid.dtype)] * N)
    av = data_valid.view  ([('', data_valid.dtype)]   * N)
    data_invalid = np.setdiff1d(ai, av).view(data_valid.dtype).reshape((-1, N))

    return prepare_binary_classification_data(data_valid, data_invalid)

# default values 

Example 25

def prepare(data):
    num = len(data)
    dim = data.shape[1]//2
    print(data.shape,num,dim)
    pre, suc = data[:,:dim], data[:,dim:]
    
    suc_invalid = np.copy(suc)
    random.shuffle(suc_invalid)
    data_invalid = np.concatenate((pre,suc_invalid),axis=1)

    ai = data_invalid.view([('', data_invalid.dtype)] * 2*dim)
    av = data.view        ([('', data.dtype)]         * 2*dim)
    data_invalid = np.setdiff1d(ai, av).view(data_invalid.dtype).reshape((-1, 2*dim))
    
    inputs = np.concatenate((data,data_invalid),axis=0)
    outputs = np.concatenate((np.ones((num,1)),np.zeros((len(data_invalid),1))),axis=0)
    print(inputs.shape,outputs.shape)
    io = np.concatenate((inputs,outputs),axis=1)
    random.shuffle(io)

    train_n = int(2*num*0.9)
    train, test = io[:train_n], io[train_n:]
    train_in, train_out = train[:,:dim*2], train[:,dim*2:]
    test_in, test_out = test[:,:dim*2], test[:,dim*2:]
    
    return train_in, train_out, test_in, test_out
    

# default values 

Example 26

def set_difference(a, b):
    assert a.shape[1:] == b.shape[1:]
    a = a.copy()
    b = b.copy()
    a_v = a.view([('', a.dtype)] * a.shape[1])
    b_v = b.view([('', b.dtype)] * b.shape[1])
    return np.setdiff1d(a_v, b_v).view(a.dtype).reshape((-1, a.shape[1])) 

Example 27

def test_indices(self, test_indices):
        if test_indices is None:
            self._train_indices = np.arange(0, len(self.y))
        else:
            self._test_indices = test_indices
            self._train_indices = np.setdiff1d(np.arange(0, len(self.y)), self.test_indices) 

Example 28

def setdiff1d(ar1, ar2, assume_unique=False):
    """
    Find the set difference of two arrays.

    Return the sorted, unique values in `ar1` that are not in `ar2`.

    Parameters
    ----------
    ar1 : array_like
        Input array.
    ar2 : array_like
        Input comparison array.
    assume_unique : bool
        If True, the input arrays are both assumed to be unique, which
        can speed up the calculation.  Default is False.

    Returns
    -------
    setdiff1d : ndarray
        Sorted 1D array of values in `ar1` that are not in `ar2`.

    See Also
    --------
    numpy.lib.arraysetops : Module with a number of other functions for
                            performing set operations on arrays.

    Examples
    --------
    >>> a = np.array([1, 2, 3, 2, 4, 1])
    >>> b = np.array([3, 4, 5, 6])
    >>> np.setdiff1d(a, b)
    array([1, 2])

    """
    if assume_unique:
        ar1 = np.asarray(ar1).ravel()
    else:
        ar1 = unique(ar1)
        ar2 = unique(ar2)
    return ar1[in1d(ar1, ar2, assume_unique=True, invert=True)] 

Example 29

def stabilize(self, prior_columns, percent):
        """
        This activates prior columns to force active in order to maintain
        the given percent of column overlap between time steps.  Always call
        this between compute and learn!
        """
        # num_active      = (len(self.columns) + len(prior_columns)) / 2
        num_active      = len(self.columns)
        overlap         = self.columns.overlap(prior_columns)
        stabile_columns = int(round(num_active * overlap))
        target_columns  = int(round(num_active * percent))
        add_columns     = target_columns - stabile_columns
        if add_columns <= 0:
            return

        eligable_columns  = np.setdiff1d(prior_columns.flat_index, self.columns.flat_index)
        eligable_excite   = self.raw_excitment[eligable_columns]
        selected_col_nums = np.argpartition(-eligable_excite, add_columns-1)[:add_columns]
        selected_columns  = eligable_columns[selected_col_nums]
        selected_index    = np.unravel_index(selected_columns, self.columns.dimensions)
        # Learn.  Note: selected columns will learn twice.  The previously
        # active segments learn now, the current most excited segments in the
        # method SP.learn().
        # Or learn not at all if theres a bug in my code...
        # if self.multisegment:
        #     if hasattr(self, 'prior_segment_excitement'):
        #         segment_excitement = self.prior_segment_excitement[selected_index]
        #         seg_idx = np.argmax(segment_excitement, axis=-1)
        #         self.proximal.learn_outputs(input_sdr=input_sdr,
        #                                     output_sdr=selected_index + (seg_idx,))
        #     self.prev_segment_excitement = self.segment_excitement
        # else:
        #     1/0
        self.columns.flat_index = np.concatenate([self.columns.flat_index, selected_columns]) 

Example 30

def sample_weights(self, idxs, scores):
        N = len(scores)
        S1 = scores[np.setdiff1d(np.arange(N), idxs)].sum()

        return np.tile([float(N), float(S1)], (len(idxs), 1)) 

Example 31

def sample_weights(self, idxs, scores):
        N = len(scores)
        S1 = scores[np.setdiff1d(np.arange(N), idxs)].sum()

        return np.tile([float(N), float(S1)], (len(idxs), 1)) 

Example 32

def sample_weights(self, idxs, scores):
        N = len(scores)
        S1 = scores[np.setdiff1d(np.arange(N), idxs)].sum()

        return np.tile([float(N), float(S1)], (len(idxs), 1)) 

Example 33

def sample_weights(self, idxs, scores):
        N = len(scores)
        S1 = scores[np.setdiff1d(np.arange(N), idxs)].sum()

        return np.tile([float(N), float(S1)], (len(idxs), 1)) 

Example 34

def _generate_pairs(ids):
    id_i, id_j = np.meshgrid(ids, ids, indexing='ij') # Grouping the input object rois
    id_i = id_i.reshape(-1) 
    id_j = id_j.reshape(-1)
    # remove the diagonal items
    id_num = len(ids)
    diagonal_items = np.array(range(id_num))
    diagonal_items = diagonal_items * id_num + diagonal_items
    all_id = range(len(id_i))
    selected_id = np.setdiff1d(all_id, diagonal_items)
    id_i = id_i[selected_id]
    id_j = id_j[selected_id]

    return id_i, id_j 

Example 35

def ttv(self, v, modes=[], without=False):
        """
        Tensor times vector product

        Parameters
        ----------
        v : 1-d array or tuple of 1-d arrays
            Vector to be multiplied with tensor.
        modes : array_like of integers, optional
            Modes in which the vectors should be multiplied.
        without : boolean, optional
            If True, vectors are multiplied in all modes **except** the
            modes specified in ``modes``.

        """
        if not isinstance(v, tuple):
            v = (v, )
        dims, vidx = check_multiplication_dims(modes, self.ndim, len(v), vidx=True, without=without)
        for i in range(len(dims)):
            if not len(v[vidx[i]]) == self.shape[dims[i]]:
                raise ValueError('Multiplicant is wrong size')
        remdims = np.setdiff1d(range(self.ndim), dims)
        return self._ttv_compute(v, dims, vidx, remdims)

    #@abstractmethod
    #def ttt(self, other, modes=None):
    #    pass 

Example 36

def setdiff1d(ar1, ar2, assume_unique=False):
    """
    Find the set difference of two arrays.

    Return the sorted, unique values in `ar1` that are not in `ar2`.

    Parameters
    ----------
    ar1 : array_like
        Input array.
    ar2 : array_like
        Input comparison array.
    assume_unique : bool
        If True, the input arrays are both assumed to be unique, which
        can speed up the calculation.  Default is False.

    Returns
    -------
    setdiff1d : ndarray
        Sorted 1D array of values in `ar1` that are not in `ar2`.

    See Also
    --------
    numpy.lib.arraysetops : Module with a number of other functions for
                            performing set operations on arrays.

    Examples
    --------
    >>> a = np.array([1, 2, 3, 2, 4, 1])
    >>> b = np.array([3, 4, 5, 6])
    >>> np.setdiff1d(a, b)
    array([1, 2])

    """
    if assume_unique:
        ar1 = np.asarray(ar1).ravel()
    else:
        ar1 = unique(ar1)
        ar2 = unique(ar2)
    return ar1[in1d(ar1, ar2, assume_unique=True, invert=True)] 

Example 37

def test_boolean_spheres_overlap():
    r"""Test to make sure that boolean objects (spheres, overlap)
    behave the way we expect.

    Test overlapping spheres.
    """
    ds = fake_amr_ds()
    sp1 = ds.sphere([0.45, 0.45, 0.45], 0.15)
    sp2 = ds.sphere([0.55, 0.55, 0.55], 0.15)
    # Get indices of both.
    i1 = sp1["index","morton_index"]
    i2 = sp2["index","morton_index"]
    # Make some booleans
    bo1 = sp1 & sp2
    bo2 = sp1 - sp2
    bo3 = sp1 | sp2
    bo4 = ds.union([sp1, sp2])
    bo5 = ds.intersection([sp1, sp2])
    # Now make sure the indices also behave as we expect.
    lens = np.intersect1d(i1, i2)
    apple = np.setdiff1d(i1, i2)
    both = np.union1d(i1, i2)
    b1 = bo1["index","morton_index"]
    b1.sort()
    b2 = bo2["index","morton_index"]
    b2.sort()
    b3 = bo3["index","morton_index"]
    b3.sort()
    assert_array_equal(b1, lens)
    assert_array_equal(b2, apple)
    assert_array_equal(b3, both)
    b4 = bo4["index","morton_index"]
    b4.sort()
    b5 = bo5["index","morton_index"]
    b5.sort()
    assert_array_equal(b3, b4)
    assert_array_equal(b1, b5)
    bo6 = sp1 ^ sp2
    b6 = bo6["index", "morton_index"]
    b6.sort()
    assert_array_equal(b6, np.setxor1d(i1, i2)) 

Example 38

def test_boolean_regions_overlap():
    r"""Test to make sure that boolean objects (regions, overlap)
    behave the way we expect.

    Test overlapping regions.
    """
    ds = fake_amr_ds()
    re1 = ds.region([0.55]*3, [0.5]*3, [0.6]*3)
    re2 = ds.region([0.6]*3, [0.55]*3, [0.65]*3)
    # Get indices of both.
    i1 = re1["index","morton_index"]
    i2 = re2["index","morton_index"]
    # Make some booleans
    bo1 = re1 & re2
    bo2 = re1 - re2
    bo3 = re1 | re2
    bo4 = ds.union([re1, re2])
    bo5 = ds.intersection([re1, re2])
    # Now make sure the indices also behave as we expect.
    cube = np.intersect1d(i1, i2)
    bite_cube = np.setdiff1d(i1, i2)
    both = np.union1d(i1, i2)
    b1 = bo1["index","morton_index"]
    b1.sort()
    b2 = bo2["index","morton_index"]
    b2.sort()
    b3 = bo3["index","morton_index"]
    b3.sort()
    assert_array_equal(b1, cube)
    assert_array_equal(b2, bite_cube)
    assert_array_equal(b3, both)
    b4 = bo4["index","morton_index"]
    b4.sort()
    b5 = bo5["index","morton_index"]
    b5.sort()
    assert_array_equal(b3, b4)
    assert_array_equal(b1, b5)
    bo6 = re1 ^ re2
    b6 = bo6["index", "morton_index"]
    b6.sort()
    assert_array_equal(b6, np.setxor1d(i1, i2)) 

Example 39

def test_boolean_ellipsoids_overlap():
    r"""Test to make sure that boolean objects (ellipsoids, overlap)
    behave the way we expect.

    Test overlapping ellipsoids.
    """
    ds = fake_amr_ds()
    ell1 = ds.ellipsoid([0.45]*3, 0.05, 0.05, 0.05, np.array([0.1]*3), 0.1)
    ell2 = ds.ellipsoid([0.55]*3, 0.05, 0.05, 0.05, np.array([0.1]*3), 0.1)
    # Get indices of both.
    i1 = ell1["index","morton_index"]
    i2 = ell2["index","morton_index"]
    # Make some booleans
    bo1 = ell1 & ell2
    bo2 = ell1 - ell2
    bo3 = ell1 | ell2
    bo4 = ds.union([ell1, ell2])
    bo5 = ds.intersection([ell1, ell2])
    # Now make sure the indices also behave as we expect.
    overlap = np.intersect1d(i1, i2)
    diff = np.setdiff1d(i1, i2)
    both = np.union1d(i1, i2)
    b1 = bo1["index","morton_index"]
    b1.sort()
    b2 = bo2["index","morton_index"]
    b2.sort()
    b3 = bo3["index","morton_index"]
    b3.sort()
    assert_array_equal(b1, overlap)
    assert_array_equal(b2, diff)
    assert_array_equal(b3, both)
    b4 = bo4["index","morton_index"]
    b4.sort()
    b5 = bo5["index","morton_index"]
    b5.sort()
    assert_array_equal(b3, b4)
    assert_array_equal(b1, b5)
    bo6 = ell1 ^ ell2
    b6 = bo6["index", "morton_index"]
    b6.sort()
    assert_array_equal(b6, np.setxor1d(i1, i2)) 

Example 40

def test_boolean_slices_overlap():
    r"""Test to make sure that boolean objects (slices, overlap)
    behave the way we expect.

    Test overlapping slices.
    """
    ds = fake_amr_ds()
    sl1 = ds.r[:,:,0.25]
    sl2 = ds.r[:,0.75,:]
    # Get indices of both.
    i1 = sl1["index","morton_index"]
    i2 = sl2["index","morton_index"]
    # Make some booleans
    bo1 = sl1 & sl2
    bo2 = sl1 - sl2
    bo3 = sl1 | sl2
    bo4 = ds.union([sl1, sl2])
    bo5 = ds.intersection([sl1, sl2])
    # Now make sure the indices also behave as we expect.
    line = np.intersect1d(i1, i2)
    orig = np.setdiff1d(i1, i2)
    both = np.union1d(i1, i2)
    b1 = bo1["index","morton_index"]
    b1.sort()
    b2 = bo2["index","morton_index"]
    b2.sort()
    b3 = bo3["index","morton_index"]
    b3.sort()
    assert_array_equal(b1, line)
    assert_array_equal(b2, orig)
    assert_array_equal(b3, both)
    b4 = bo4["index","morton_index"]
    b4.sort()
    b5 = bo5["index","morton_index"]
    b5.sort()
    assert_array_equal(b3, b4)
    assert_array_equal(b1, b5)
    bo6 = sl1 ^ sl2
    b6 = bo6["index", "morton_index"]
    b6.sort()
    assert_array_equal(b6, np.setxor1d(i1, i2)) 

Example 41

def analyze_false(validData,validDataNumbers,validLabels,model):	
	'Calculating precision and recall for best model...'
	predictions = np.squeeze((model.predict(validDataNumbers) > 0.5).astype('int32'))
	c1_inds = np.where(validLabels == 1)[0]
	pos_inds = np.where((predictions+validLabels) == 2)[0] #np.squeeze(predictions) == validLabels
	neg_inds = np.setdiff1d(c1_inds,pos_inds)
	seq_lengths = np.zeros((validData.shape[0]))
	for ind,row in np.ndenumerate(validData):
	        seq_lengths[ind] = len(wordpunct_tokenize(row.lower().strip()))	

	mean_true_length = np.mean(seq_lengths[pos_inds])	
	mean_false_length = np.mean(seq_lengths[neg_inds])
	
	return mean_false_length,mean_true_length 

Example 42

def _get_epochs_interpolation(self, epochs, drop_log,
                                  ch_type, verbose='progressbar'):
        """Interpolate the bad epochs."""
        # 1: bad segment, # 2: interpolated
        fix_log = drop_log.copy()
        ch_names = epochs.ch_names
        non_picks = np.setdiff1d(range(epochs.info['nchan']), self.picks)
        interp_channels = list()
        n_interpolate = self.n_interpolate[ch_type]
        for epoch_idx in range(len(epochs)):
            n_bads = drop_log[epoch_idx, self.picks].sum()
            if n_bads == 0:
                continue
            else:
                if n_bads <= n_interpolate:
                    interp_chs_mask = drop_log[epoch_idx] == 1
                else:
                    # get peak-to-peak for channels in that epoch
                    data = epochs[epoch_idx].get_data()[0]
                    peaks = np.ptp(data, axis=-1)
                    peaks[non_picks] = -np.inf
                    # find channels which are bad by rejection threshold
                    interp_chs_mask = drop_log[epoch_idx] == 1
                    # ignore good channels
                    peaks[~interp_chs_mask] = -np.inf
                    # find the ordering of channels amongst the bad channels
                    sorted_ch_idx_picks = np.argsort(peaks)[::-1]
                    # then select only the worst n_interpolate channels
                    interp_chs_mask[
                        sorted_ch_idx_picks[n_interpolate:]] = False

            fix_log[epoch_idx][interp_chs_mask] = 2
            interp_chs = np.where(interp_chs_mask)[0]
            interp_chs = [ch_name for idx, ch_name in enumerate(ch_names)
                          if idx in interp_chs]
            interp_channels.append(interp_chs)
        return interp_channels, fix_log 

Example 43

def comprz_dB(xx,fr=0.05):
    """ Compress signal in such a way that is logarithmic but also avoids negative values """
    x = numpy.copy(xx)
    sh = xx.shape
    x = x.reshape(-1)
    x = comprz(x)
    x = numpy.setdiff1d(x,numpy.array([0.0]))
    xs = numpy.sort(x)
    mini = xs[int(fr*len(x))]
    mn = numpy.ones_like(xx)*mini
    xx = numpy.where(xx > mini, xx, mn)
    xx = xx.reshape(sh)
    return(10.0*numpy.log10(xx)) 

Example 44

def random_balanced_partitions(data, first_size, labels, random=np.random):
    """Split data into a balanced random partition and the rest

    Partition the `data` array into two random partitions, using
    the `labels` array (of equal size) to guide the choice of
    elements of the first returned array.

    Example:
        random_balanced_partition(['a', 'b', 'c'], 2, [3, 5, 5])
        # Both labels 3 and 5 need to be presented once, so
        # the result can be either (['a', 'b'], ['c']) or
        # (['a', 'c'], ['b']) but not (['b', 'c'], ['a']).

    Args:
        data (ndarray): data to be split
        first_size (int): size of the first partition
        balance (ndarray): according to which balancing is done
        random (RandomState): source of randomness

    Return:
        tuple of two ndarrays
    """
    assert len(data) == len(labels)

    classes, class_counts = np.unique(labels, return_counts=True)
    assert len(classes) <= 10000, "surprisingly many classes: {}".format(len(classes))
    assert first_size % len(classes) == 0, "not divisible: {}/{}".format(first_size, len(classes))
    assert np.all(class_counts >= first_size // len(classes)), "not enough examples of some class"

    idxs_per_class = [np.nonzero(labels == klass)[0] for klass in classes]
    chosen_idxs_per_class = [
        random.choice(idxs, first_size // len(classes), replace=False)
        for idxs in idxs_per_class
    ]
    first_idxs = np.concatenate(chosen_idxs_per_class)
    second_idxs = np.setdiff1d(np.arange(len(labels)), first_idxs)

    assert first_idxs.shape == (first_size,)
    assert second_idxs.shape == (len(data) - first_size,)
    return data[first_idxs], data[second_idxs] 

Example 45

def least_squares_multipliers(self, x):
        """Compute least-squares multipliers estimates."""
        al_model = self.model
        slack_model = self.model.model
        m = slack_model.m
        n = slack_model.n

        lim = max(2 * m, 2 * n)
        J = slack_model.jop(x)

        # Determine which bounds are active to remove appropriate columns of J
        on_bound = self.get_active_bounds(x,
                                          slack_model.Lvar,
                                          slack_model.Uvar)
        free_vars = np.setdiff1d(np.arange(n, dtype=np.int), on_bound)
        Jred = ReducedJacobian(J, np.arange(m, dtype=np.int),
                               free_vars)

        g = slack_model.grad(x) - J.T * al_model.pi

        lsqr = LSQRSolver(Jred.T)
        lsqr.solve(g[free_vars], itnlim=lim)
        if lsqr.optimal:
            al_model.pi += lsqr.x.copy()
        else:
            self.log.debug("lsqr failed to converge")
        return 

Example 46

def _open_file(self):
        # only apply _skip property at the beginning of the trajectory
        skip = self._data_source._skip[self._itraj] + self.skip if self._t == 0 else 0
        nt = self._data_source._skip[self._itraj] + self._data_source._lengths[self._itraj]

        # calculate an index set, which rows to skip (includes stride)
        skip_rows = np.empty(0)

        if skip > 0:
            skip_rows = np.zeros(nt)
            skip_rows[:skip] = np.arange(skip)

        if not self.uniform_stride:
            all_frames = np.arange(nt)
            skip_rows = np.setdiff1d(all_frames, self.ra_indices_for_traj(self._itraj), assume_unique=True)
        elif self.stride > 1:
            all_frames = np.arange(nt)
            if skip_rows is not None:
                wanted_frames = np.arange(skip, nt, self.stride)
            else:
                wanted_frames = np.arange(0, nt, self.stride)
            skip_rows = np.setdiff1d(
                all_frames, wanted_frames, assume_unique=True)

        self._skip_rows = skip_rows
        try:
            fh = open(self._data_source.filenames[self._itraj],
                      mode=self._data_source.DEFAULT_OPEN_MODE)
            self._file_handle = fh
        except EnvironmentError:
            self._logger.exception()
            raise 

Example 47

def setdiff1d(ar1, ar2, assume_unique=False):
    """
    Find the set difference of two arrays.

    Return the sorted, unique values in `ar1` that are not in `ar2`.

    Parameters
    ----------
    ar1 : array_like
        Input array.
    ar2 : array_like
        Input comparison array.
    assume_unique : bool
        If True, the input arrays are both assumed to be unique, which
        can speed up the calculation.  Default is False.

    Returns
    -------
    setdiff1d : ndarray
        Sorted 1D array of values in `ar1` that are not in `ar2`.

    See Also
    --------
    numpy.lib.arraysetops : Module with a number of other functions for
                            performing set operations on arrays.

    Examples
    --------
    >>> a = np.array([1, 2, 3, 2, 4, 1])
    >>> b = np.array([3, 4, 5, 6])
    >>> np.setdiff1d(a, b)
    array([1, 2])

    """
    if assume_unique:
        ar1 = np.asarray(ar1).ravel()
    else:
        ar1 = unique(ar1)
        ar2 = unique(ar2)
    return ar1[in1d(ar1, ar2, assume_unique=True, invert=True)] 

Example 48

def load_boston_data(prop=400/506):
    from sklearn import datasets
    boston = datasets.load_boston()
    X, y = boston.data, boston.target
    y = y[:, None]
    ntrain = y.shape[0]
    train_inds = npr.choice(range(ntrain), int(prop*ntrain), replace=False)
    valid_inds = np.setdiff1d(range(ntrain), train_inds)
    X_train, y_train = X[train_inds].copy(), y[train_inds].copy()
    X_valid, y_valid = X[valid_inds].copy(), y[valid_inds].copy()
    return X_train, y_train, X_valid, y_valid

############################ Training Phase ############################ 

Example 49

def _load_cv_data(self, list_files):
        """Load training and cross-validation sets."""
        # Split files for training and validation sets
        val_files = np.array_split(list_files, self.n_folds)
        train_files = np.setdiff1d(list_files, val_files[self.fold_idx])

        # Load a npz file
        print "Load training set:"
        data_train, label_train = self._load_npz_list_files(train_files)
        print " "
        print "Load validation set:"
        data_val, label_val = self._load_npz_list_files(val_files[self.fold_idx])
        print " "

        # Reshape the data to match the input of the model - conv2d
        data_train = np.squeeze(data_train)
        data_val = np.squeeze(data_val)
        data_train = data_train[:, :, np.newaxis, np.newaxis]
        data_val = data_val[:, :, np.newaxis, np.newaxis]

        # Casting
        data_train = data_train.astype(np.float32)
        label_train = label_train.astype(np.int32)
        data_val = data_val.astype(np.float32)
        label_val = label_val.astype(np.int32)

        return data_train, label_train, data_val, label_val 

Example 50

def _load_cv_data(self, list_files):
        """Load sequence training and cross-validation sets."""
        # Split files for training and validation sets
        val_files = np.array_split(list_files, self.n_folds)
        train_files = np.setdiff1d(list_files, val_files[self.fold_idx])

        # Load a npz file
        print "Load training set:"
        data_train, label_train = self._load_npz_list_files(train_files)
        print " "
        print "Load validation set:"
        data_val, label_val = self._load_npz_list_files(val_files[self.fold_idx])
        print " "

        return data_train, label_train, data_val, label_val 
点赞

发表评论

电子邮件地址不会被公开。 必填项已用*标注