From b0e9d66690895c4f50588c3988e979577198adc9 Mon Sep 17 00:00:00 2001 From: Python3pkg Date: Sun, 21 May 2017 02:12:36 -0700 Subject: [PATCH] Convert to Python3 --- coretracker/FisherExact/Fisher.py | 16 +- coretracker/FisherExact/__init__.py | 2 +- coretracker/classifier/__init__.py | 2 +- coretracker/classifier/classifier.py | 84 ++++----- coretracker/classifier/models/__init__.py | 4 +- coretracker/coreutils/AncestralRecon.py | 24 ++- coretracker/coreutils/Faces.py | 12 +- coretracker/coreutils/RNAEditChecker.py | 8 +- coretracker/coreutils/__init__.py | 10 +- coretracker/coreutils/corefile.py | 12 +- coretracker/coreutils/letterconfig.py | 2 +- coretracker/coreutils/mtgenes.py | 10 +- coretracker/coreutils/pdfutils.py | 2 +- coretracker/coreutils/template.py | 2 +- coretracker/coreutils/utils.py | 213 +++++++++++----------- coretracker/settings/__init__.py | 4 +- coretracker/settings/settings.py | 4 +- ez_setup.py | 2 +- scripts/RF/classifierdata.py | 18 +- scripts/RF/rfdataparser.py | 2 +- scripts/gendcoder_req.py | 8 +- setup.py | 8 +- 22 files changed, 223 insertions(+), 226 deletions(-) diff --git a/coretracker/FisherExact/Fisher.py b/coretracker/FisherExact/Fisher.py index 46af003..bddf43f 100644 --- a/coretracker/FisherExact/Fisher.py +++ b/coretracker/FisherExact/Fisher.py @@ -1,9 +1,9 @@ import scipy.stats as ss from scipy.special import gammaln as lgamma -import statlib.fexact as f -from statlib.fexact import fisher_exact as f_exact -from statlib.asa159 import rcont2 -from statlib.asa205 import enum as rcont +from . import statlib.fexact as f +from .statlib.fexact import fisher_exact as f_exact +from .statlib.asa159 import rcont2 +from .statlib.asa205 import enum as rcont import numpy as np import logging import os @@ -220,11 +220,11 @@ def _fisher_sim(c, replicate, seed=None): results = np.zeros(replicate) fact = np.zeros(n + 1) - for i in xrange(2, n + 1): + for i in range(2, n + 1): fact[i] = fact[i - 1] + np.log(i) observed = np.zeros((nr, nc), dtype="int32", order='F') - for it in xrange(replicate): + for it in range(replicate): rcont2(nrow=nr, ncol=nc, nrowt=sr, ncolt=sc, key=key, seed=seed, matrix=observed, ierror=ierror) # if we do not have an error, make spcial action @@ -232,7 +232,7 @@ def _fisher_sim(c, replicate, seed=None): tmp_observed = observed.ravel() if ierror[0] != 0: raise ValueError("Fortran subroutine rcont2 return an error !") - for j in xrange(nc): + for j in range(nc): i = 0 ii = j * nr while(i < nr): @@ -268,7 +268,7 @@ def _midp(c): global result result = [] logfact = np.zeros(n + 1) - for i in xrange(2, n + 1): + for i in range(2, n + 1): logfact[i] = logfact[i - 1] + np.log(i) def callback(iflag, table, m, n, rowsum, colsum, prob, mult): diff --git a/coretracker/FisherExact/__init__.py b/coretracker/FisherExact/__init__.py index a638f65..df2fdba 100644 --- a/coretracker/FisherExact/__init__.py +++ b/coretracker/FisherExact/__init__.py @@ -1,2 +1,2 @@ -from Fisher import fisher_exact +from .Fisher import fisher_exact __all__ = [fisher_exact] diff --git a/coretracker/classifier/__init__.py b/coretracker/classifier/__init__.py index b75f5f7..96198d2 100644 --- a/coretracker/classifier/__init__.py +++ b/coretracker/classifier/__init__.py @@ -11,7 +11,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from classifier import Classifier, getDataFromFeatures, read_from_json +from .classifier import Classifier, getDataFromFeatures, read_from_json import os this_dir, this_filename = os.path.split(__file__) MODELPATH = os.path.join(this_dir, "models", '%s/classifier.pkl.z') diff --git a/coretracker/classifier/classifier.py b/coretracker/classifier/classifier.py index 83245b8..7a27ace 100755 --- a/coretracker/classifier/classifier.py +++ b/coretracker/classifier/classifier.py @@ -1,4 +1,4 @@ -from __future__ import division + import itertools import json @@ -55,7 +55,7 @@ def load_from_file(clc, loadfile): clf = joblib.load(loadfile) return clf except IOError: - print('Problem with file %s, can not open it' % loadfile) + print(('Problem with file %s, can not open it' % loadfile)) except Exception as e: raise e return None @@ -112,11 +112,11 @@ def feature_importance(self, outfile="importance.png", features_list=[]): [tree.feature_importances_ for tree in self.clf.estimators_], axis=0) plt.figure() plt.title("Feature importances") - plt.bar(range(n_feats), importances[ + plt.bar(list(range(n_feats)), importances[ indices], width=0.5, color="b", yerr=std[indices], align="center") if len(features_list) > 0: features_list = np.asarray(features_list)[indices] - plt.xticks(range(n_feats), features_list, rotation='vertical') + plt.xticks(list(range(n_feats)), features_list, rotation='vertical') plt.xlim([-1, n_feats]) plt.margins(0.2) @@ -176,21 +176,21 @@ def get_stat(self, X_test, y_test): (prob_pos.max() - prob_pos.min()) clf_score = brier_score_loss(y_test, prob_pos) - print("%s:" % self.method) - print("\tBrier: %1.3f" % (clf_score)) - print("\tPrecision: %1.3f" % precision_score(y_test, y_pred)) - print("\tRecall: %1.3f" % recall_score(y_test, y_pred)) - print("\tF1: %1.3f" % f1_score(y_test, y_pred)) - print("\tROC AUC score: %1.3f\n" % roc_auc_score(y_test, prob_pos)) + print(("%s:" % self.method)) + print(("\tBrier: %1.3f" % (clf_score))) + print(("\tPrecision: %1.3f" % precision_score(y_test, y_pred))) + print(("\tRecall: %1.3f" % recall_score(y_test, y_pred))) + print(("\tF1: %1.3f" % f1_score(y_test, y_pred))) + print(("\tROC AUC score: %1.3f\n" % roc_auc_score(y_test, prob_pos))) def read_from_json(data, labels=None, use_global=True, use_pvalue=True): """Parse X array from data""" - if isinstance(data, basestring): + if isinstance(data, str): with open(data) as jfile: data = json.load(jfile) - if labels and isinstance(labels, basestring): + if labels and isinstance(labels, str): with open(labels) as jfile2: labels = json.load(jfile2) # matrice format @@ -209,9 +209,9 @@ def read_from_json(data, labels=None, use_global=True, use_pvalue=True): # each entry format : # [fitch, suspected, gene_frac, rea_frac, used_frac, subs_count, codon_lik_for_rea_aa] - for aa2, val in data['aa'].items(): - for aa1, glist in val.items(): - for genome, gdata in glist.items(): + for aa2, val in list(data['aa'].items()): + for aa1, glist in list(val.items()): + for genome, gdata in list(glist.items()): type_check = gdata[dtype] codon_total = gdata['codons'][dtype] fitch = gdata['fitch'] @@ -221,10 +221,10 @@ def read_from_json(data, labels=None, use_global=True, use_pvalue=True): used_codon = type_check['used_codon'] # gene_in_genome = data['genes'][genome] was_lost = gdata['lost'][fisher_type] - total_aa = np.sum(codon_total.values()) + total_aa = np.sum(list(codon_total.values())) # mixte_codon = type_check['mixte_codon'] subs_count = type_check['count'] - for codon in codon_total.keys(): + for codon in list(codon_total.keys()): gene_count = 0 total_gene_count = 0 try: @@ -320,7 +320,7 @@ def get_2D_distinct(Xdata, Xlabel, y, etiquette, outfile="2Dcompare.png", featur ncomp = len(features) if ncomp == 0 or ncomp > len(etiquette): ncomp = len(etiquette) - features = range(len(etiquette)) + features = list(range(len(etiquette))) else: Xdata = Xdata[:, features] @@ -331,8 +331,8 @@ def get_2D_distinct(Xdata, Xlabel, y, etiquette, outfile="2Dcompare.png", featur plt.close('all') f, axarr = plt.subplots(i, j) - for xax in xrange(ncomp): - for yax in xrange(xax + 1, ncomp): + for xax in range(ncomp): + for yax in range(xax + 1, ncomp): total_size -= 1 i, j = np.unravel_index(total_size, axarr.shape) axarr[i, j].scatter(Xdata[:, xax], Xdata[:, yax], c=color) @@ -385,8 +385,8 @@ def draw_pca_data(X_features, Xlabel, y, outfile="PCA.png"): plt.close('all') f, axarr = plt.subplots(i, j) if total_size > 1: - for xax in xrange(ncomp): - for yax in xrange(xax + 1, ncomp): + for xax in range(ncomp): + for yax in range(xax + 1, ncomp): total_size -= 1 i, j = np.unravel_index(total_size, axarr.shape) axarr[i, j].scatter(X_features[:, xax], @@ -413,11 +413,11 @@ def print_data(X, X_label, Y, etiquette=None): "N. used", "Cod. count", "Sub. count", "G. len", "codon_lik", "N. mixte", "id"] etiquette = list(etiquette) - print("\n" + "\t".join(["genome", "codon", - "ori_aa", "rea_aa"] + etiquette)) - for i in xrange(len(X_label)): + print(("\n" + "\t".join(["genome", "codon", + "ori_aa", "rea_aa"] + etiquette))) + for i in range(len(X_label)): if Y[i] == 1: - print("\t".join(list(X_label[i]) + [str(x) for x in X[i]])) + print(("\t".join(list(X_label[i]) + [str(x) for x in X[i]]))) def getDataFromFeatures(Xdata, etiquette, feats=[]): @@ -434,7 +434,7 @@ def get_sensibility_and_precision(pred_y, true_y, X_labels=None, X=None, log=Tru assert nel == len(pred_y), 'Vector should be the same size\n' true_pos, true_neg, false_pos, false_neg = 0.0, 0.0, 0.0, 0.0 false_neg_list, false_pos_list = [], [] - for i in xrange(len(pred_y)): + for i in range(len(pred_y)): if pred_y[i] == 0 and true_y[i] == 1: false_neg += 1 false_neg_list.append(i) @@ -446,30 +446,30 @@ def get_sensibility_and_precision(pred_y, true_y, X_labels=None, X=None, log=Tru elif pred_y[i] == 0 and true_y[i] == 0: true_neg += 1 - print("Test size is: %d\nTrue Positive is: %d\nTrue negative is: \ + print(("Test size is: %d\nTrue Positive is: %d\nTrue negative is: \ %d\nFalse positive is: %d\nFalse negative is:%d" % ( - nel, true_pos, true_neg, false_pos, false_neg)) + nel, true_pos, true_neg, false_pos, false_neg))) print('-------------------------------------------') - print("Sensibility is %f" % (true_pos / (true_pos + false_neg) - if (true_pos + false_neg) > 0 else 1)) - print("Specificity is %f" % (true_neg / (true_neg + false_pos) - if (true_neg + false_pos) > 0 else 1)) - print("Accuracy is %f" % ((true_neg + true_pos) / nel)) - print("Precision is %f\n\n" % - (true_pos / (true_pos + false_pos) if (true_pos + false_pos) > 0 else 1)) + print(("Sensibility is %f" % (true_pos / (true_pos + false_neg) + if (true_pos + false_neg) > 0 else 1))) + print(("Specificity is %f" % (true_neg / (true_neg + false_pos) + if (true_neg + false_pos) > 0 else 1))) + print(("Accuracy is %f" % ((true_neg + true_pos) / nel))) + print(("Precision is %f\n\n" % + (true_pos / (true_pos + false_pos) if (true_pos + false_pos) > 0 else 1))) if log: if X_labels is not None and X is not None: if len(false_neg_list) > 0: print("List of false negatives") for i in false_neg_list: - print("\t".join(X_labels[i])) - print("\t".join([str(x) for x in X[i]])) + print(("\t".join(X_labels[i]))) + print(("\t".join([str(x) for x in X[i]]))) if len(false_pos_list) > 0: print("\nList of False positives") for i in false_pos_list: - print("\t".join(X_labels[i])) - print("\t".join([str(x) for x in X[i]])) + print(("\t".join(X_labels[i]))) + print(("\t".join([str(x) for x in X[i]]))) def split_zeros_pos(L, X, Y, split_size=300): @@ -493,7 +493,7 @@ def get_aa_cross_val(L, X, Y, AA, tsize=None, rstate=-1): """Get test data from dataset""" test_position = [] aa_y = np.zeros(Y.shape) - for i in xrange(len(Y)): + for i in range(len(Y)): if L[i][-1] == AA: aa_y[i] = 1 test_position.append(i) @@ -510,7 +510,7 @@ def get_aa_cross_val(L, X, Y, AA, tsize=None, rstate=-1): test_position = np.random.permutation(test_position) mask = np.ones(Y.shape, dtype=bool) mask[test_position] = False - train_position = np.array(range(len(mask)))[mask] + train_position = np.array(list(range(len(mask))))[mask] if rstate > 0: return shuffle(train_position, random_state=rstate), shuffle(test_position, random_state=rstate) diff --git a/coretracker/classifier/models/__init__.py b/coretracker/classifier/models/__init__.py index e84d27a..0ed24b9 100644 --- a/coretracker/classifier/models/__init__.py +++ b/coretracker/classifier/models/__init__.py @@ -23,7 +23,7 @@ class ModelType(object): default_sfeat = {'1': mod1, '2': mod2, '3': mod3} def __init__(self, m, etiquette, sfeat=[], encode=False): - if m not in self.default_sfeat.keys(): + if m not in list(self.default_sfeat.keys()): raise ValueError('Selected model do not exist') self.model = str(m) self.etiquette = etiquette @@ -31,7 +31,7 @@ def __init__(self, m, etiquette, sfeat=[], encode=False): try: self.sfeat = self.default_sfeat[self.model] except: - self.sfeat = range(len(etiquette)) + self.sfeat = list(range(len(etiquette))) else: self.sfeat = sfeat self.encode = encode diff --git a/coretracker/coreutils/AncestralRecon.py b/coretracker/coreutils/AncestralRecon.py index 91eda27..6eddcb2 100644 --- a/coretracker/coreutils/AncestralRecon.py +++ b/coretracker/coreutils/AncestralRecon.py @@ -1,22 +1,20 @@ from abc import ABCMeta, abstractmethod -import utils +from . import utils from collections import defaultdict, Counter import numpy as np import operator -from letterconfig import * +from .letterconfig import * def init_back_table(dct): """Get back table for the current genetic code""" back_table = defaultdict(list) - for aa, codon in zip(dct.forward_table.values(), dct.forward_table.keys()): + for aa, codon in zip(list(dct.forward_table.values()), list(dct.forward_table.keys())): back_table[aa].append(codon) return back_table -class AbsAncest: - __metaclass__ = ABCMeta - +class AbsAncest(metaclass=ABCMeta): def __init__(self, tree, nodestates): self.tree = tree self.nodestates = nodestates @@ -55,8 +53,8 @@ def flip_rea_forward(clc, nodestates): """Flip rea data dict""" new_dt = utils.makehash(1, set) state_map = defaultdict(set) - for (genome, aarea) in nodestates.items(): - nl = [(c, aa) for aa, codons in aarea.items() for c in codons] + for (genome, aarea) in list(nodestates.items()): + nl = [(c, aa) for aa, codons in list(aarea.items()) for c in codons] for (c, aa) in nl: new_dt[genome][c].add(aa) state_map[c].add(aa) @@ -149,13 +147,13 @@ def __const_term_state(self, term_list, smat, char_states, nullstate): # sort, whether by total number if self.enforce_order: - return [(x, is_valid_for_c[x]) for x in char_states if x in is_valid_for_c.keys()] + return [(x, is_valid_for_c[x]) for x in char_states if x in list(is_valid_for_c.keys())] if self.sort_by_size: - possible_order = sorted([(x, np.count_nonzero(y)) for x, y in is_valid_for_c.items( - )], key=operator.itemgetter(1), reverse=True) + possible_order = sorted([(x, np.count_nonzero(y)) for x, y in list(is_valid_for_c.items( + ))], key=operator.itemgetter(1), reverse=True) else: # here we sort by subtree weight, ignoring excluded subgroup - possible_order = sorted([(x, y[-1]) for x, y in is_valid_for_c.items()], + possible_order = sorted([(x, y[-1]) for x, y in list(is_valid_for_c.items())], key=operator.itemgetter(1), reverse=True) return possible_order @@ -263,7 +261,7 @@ def __init__(self, tree, reassigned, ori_aa, dest_aa, dct, codon_rea=(None, None self.dct = dct self.back_table = init_back_table(dct) codon_list = self.back_table[aa_letters_3to1[ori_aa]] - self.colors = dict(zip(codon_list, colors)) + self.colors = dict(list(zip(codon_list, colors))) for leaf in tree: if leaf.name in reassigned: leaf.add_features(reassigned={1}) diff --git a/coretracker/coreutils/Faces.py b/coretracker/coreutils/Faces.py index 2ae4811..e780f95 100644 --- a/coretracker/coreutils/Faces.py +++ b/coretracker/coreutils/Faces.py @@ -94,12 +94,12 @@ def _get_codon_fgcolors(codontable, cible_aa): """Get colon foreground color""" - return dict((k, (_aafgcolors[v] if v != cible_aa else '#FFFFFF')) for (k, v) in codontable.items()) + return dict((k, (_aafgcolors[v] if v != cible_aa else '#FFFFFF')) for (k, v) in list(codontable.items())) def _get_codon_bgcolors(codontable, cible_aa, spec_codon_col): """Get colon background color""" - return dict((k, spec_codon_col.get(k, ("#FFFFFF" if v != cible_aa else '#000000'))) for (k, v) in codontable.items()) + return dict((k, spec_codon_col.get(k, ("#FFFFFF" if v != cible_aa else '#000000'))) for (k, v) in list(codontable.items())) class PPieChartFace(faces.StaticItemFace): @@ -120,7 +120,7 @@ def __init__(self, percents, width, height, colors=None, line_color=None, label_ if not is_percent: s = sum(percents) - percents = map(lambda x: x * 100. / s, percents) + percents = [x * 100. / s for x in percents] if round(sum(percents)) > 100: raise ValueError("PPieChartItem: percentage values > 100") @@ -271,7 +271,7 @@ def __init__(self, seq, cible_aa, seqtype="aa", fsize=10, self.col_w *= 3 if not isinstance(self.seq, list): # only consider the position where 3 nuc can be obtained - self.seq = [self.seq[i:i + 3] for i in xrange(0, + self.seq = [self.seq[i:i + 3] for i in range(0, len(self.seq) - len(self.seq) % 3, 3)] if not fg_colors: fg_colors = _get_codon_fgcolors(codontable, cible_aa) @@ -421,7 +421,7 @@ def __init__(self, aalist, readict, is_leaf=True, spacer=1, height=12, def _init_colors(bgtype=True, fgcolor='#000000'): color = {} - for aa in readict.keys(): + for aa in list(readict.keys()): c = _aabgcolors[aa.upper()] if bgtype else fgcolor color[aa.upper()] = QBrush(QColor(c)) return color @@ -431,7 +431,7 @@ def _init_colors(bgtype=True, fgcolor='#000000'): def update_items(self): try: max_codons = math.ceil( - max([len(x) for x in self.readict.values()]) / 2.0) * 2 + max([len(x) for x in list(self.readict.values())]) / 2.0) * 2 except: max_codons = 1 if self.maxcodon: diff --git a/coretracker/coreutils/RNAEditChecker.py b/coretracker/coreutils/RNAEditChecker.py index b43a64e..ed4bb00 100644 --- a/coretracker/coreutils/RNAEditChecker.py +++ b/coretracker/coreutils/RNAEditChecker.py @@ -16,7 +16,7 @@ def __init__(self, subsrea, table): self.create_graph() def create_graph(self): - for codon in codontable.forward_dict.keys(): + for codon in list(codontable.forward_dict.keys()): codtrans = CodonTransition(codon, self.subsrea, self.codontable) for trans in codtrans.iter_possible_transition(): self.codon_graph[codon].append(trans) @@ -38,8 +38,8 @@ def iter_possible_transition(self): self.codon) if self.subsrea.is_edited(x)] pstates = [0, 1] for state in itertools.product([0, 1], repeat=len(expc_trans)): - state = map(lambda x: state[x] * - expc_trans[x], range(len(expc_trans))) + state = [state[x] * + expc_trans[x] for x in range(len(expc_trans))] codon = self.codon[:] changed = False for pos in state: @@ -96,7 +96,7 @@ def __init__(self, codonalign, aalign, gcode=1, reatype=('C', 'U'), radius=None) self.aalign = self._to_mutable(aalign) self.dict_align = SeqIO.to_dict(codonalign) self.alen = codonalign.get_aln_length() - self.glist = self.dict_align.keys() + self.glist = list(self.dict_align.keys()) self.radius = radius self._compute_info_content() self.reatype = SubsReaType(reatype[0], reatype[1]) diff --git a/coretracker/coreutils/__init__.py b/coretracker/coreutils/__init__.py index 5f59a04..2178949 100644 --- a/coretracker/coreutils/__init__.py +++ b/coretracker/coreutils/__init__.py @@ -12,12 +12,12 @@ # along with this program. If not, see . import warnings -from corefile import CoreFile +from .corefile import CoreFile warnings.filterwarnings("ignore") -import utils -from utils import SequenceLoader, SequenceSet, ReaGenomeFinder -import AncestralRecon -import Faces +from . import utils +from .utils import SequenceLoader, SequenceSet, ReaGenomeFinder +from . import AncestralRecon +from . import Faces __all__ = ['utils', 'SequenceLoader', 'SequenceSet', 'CoreFile', 'ReaGenomeFinder', 'AncestralRecon', 'Faces'] diff --git a/coretracker/coreutils/corefile.py b/coretracker/coreutils/corefile.py index a51b037..65472e2 100644 --- a/coretracker/coreutils/corefile.py +++ b/coretracker/coreutils/corefile.py @@ -29,7 +29,7 @@ class CoreFile: def __init__(self, infile, alphabet=generic_protein): self.infile = infile - if isinstance(alphabet, basestring): + if isinstance(alphabet, str): if alphabet.startswith('nuc'): alphabet = generic_nucleotide else: @@ -110,7 +110,7 @@ def write(self, outfile, sequences=None): @classmethod def write_corefile(clc, sequences, outfile): with open(outfile, 'w') as OUT: - for gene, sequence in sequences.items(): + for gene, sequence in list(sequences.items()): OUT.write('>>%s\n' % gene) for seq in sequence: OUT.write('>%s\n' % seq.name) @@ -126,7 +126,7 @@ def get_sequences(self): def items(self): """ iterate over the keys and values""" - return self.sequences.items() + return list(self.sequences.items()) @classmethod def split_alignment(clc, alignment, genelimit): @@ -134,7 +134,7 @@ def split_alignment(clc, alignment, genelimit): # genelimit convert: sequences = {} if isinstance(alignment, dict): - alignment = MSA(alignment.values()) + alignment = MSA(list(alignment.values())) exp_len = alignment.get_alignment_length() for dt in genelimit: gene, start, end = dt @@ -148,8 +148,8 @@ def split_alignment(clc, alignment, genelimit): def flip_data(clc, indict): """Change data structure for dict of genome to dict of genes""" flip_dt = ddict() - for (genome, genedict) in indict.items(): - for (gene, seq) in genedict.items(): + for (genome, genedict) in list(indict.items()): + for (gene, seq) in list(genedict.items()): try: flip_dt[gene][genome] += str(seq) except: diff --git a/coretracker/coreutils/letterconfig.py b/coretracker/coreutils/letterconfig.py index 37e0596..d6195c4 100644 --- a/coretracker/coreutils/letterconfig.py +++ b/coretracker/coreutils/letterconfig.py @@ -9,4 +9,4 @@ 'Y': 'Tyr', } -aa_letters_3to1 = dict((x[1], x[0]) for x in aa_letters_1to3.items()) +aa_letters_3to1 = dict((x[1], x[0]) for x in list(aa_letters_1to3.items())) diff --git a/coretracker/coreutils/mtgenes.py b/coretracker/coreutils/mtgenes.py index acf7b39..2c68cc1 100644 --- a/coretracker/coreutils/mtgenes.py +++ b/coretracker/coreutils/mtgenes.py @@ -37,7 +37,7 @@ } -revmtgenes = dict((v, k) for k in mtgenes.keys() for v in mtgenes[k]) +revmtgenes = dict((v, k) for k in list(mtgenes.keys()) for v in mtgenes[k]) try: import os curdir = os.path.dirname(os.path.realpath(__file__)) @@ -47,17 +47,17 @@ key, value = line.strip().split(None, 1) correspondances = value.split('|') key = key.lower() - if key in revmtgenes.keys(): + if key in list(revmtgenes.keys()): mtgenes[revmtgenes[key]].update(correspondances) else: reverse_find = False for c in correspondances: - if c in mtgenes.keys(): + if c in list(mtgenes.keys()): reverse_find = True mtgenes[c].add(key) if not reverse_find: mtgenes[key] = set(correspondances) except: - print "Error parsing mtgenes alias" + print("Error parsing mtgenes alias") pass -revmtgenes = dict((v, k) for k in mtgenes.keys() for v in mtgenes[k]) +revmtgenes = dict((v, k) for k in list(mtgenes.keys()) for v in mtgenes[k]) diff --git a/coretracker/coreutils/pdfutils.py b/coretracker/coreutils/pdfutils.py index ce0f87e..f2f26c6 100644 --- a/coretracker/coreutils/pdfutils.py +++ b/coretracker/coreutils/pdfutils.py @@ -1,4 +1,4 @@ -from template import BasicTemplate +from .template import BasicTemplate from weasyprint import HTML import os diff --git a/coretracker/coreutils/template.py b/coretracker/coreutils/template.py index acec563..9e07f9b 100644 --- a/coretracker/coreutils/template.py +++ b/coretracker/coreutils/template.py @@ -1,4 +1,4 @@ -from output import Output +from .output import Output class BasicTemplate(object): diff --git a/coretracker/coreutils/utils.py b/coretracker/coreutils/utils.py index 31f7381..1ffae3b 100644 --- a/coretracker/coreutils/utils.py +++ b/coretracker/coreutils/utils.py @@ -1,4 +1,4 @@ -from __future__ import division + import argparse import glob @@ -14,7 +14,7 @@ import time import traceback from collections import Counter, defaultdict -from cStringIO import StringIO +from io import StringIO from distutils import spawn from functools import partial @@ -37,13 +37,13 @@ from ete3 import Tree from scipy.cluster.vq import kmeans2 -from AncestralRecon import SingleNaiveRec, init_back_table -from corefile import CoreFile +from .AncestralRecon import SingleNaiveRec, init_back_table +from .corefile import CoreFile from coretracker.FisherExact import fisher_exact -from Faces import LineFace, List90Face, PPieChartFace, SequenceFace -from letterconfig import * -from output import Output -from pdfutils import * +from .Faces import LineFace, List90Face, PPieChartFace, SequenceFace +from .letterconfig import * +from .output import Output +from .pdfutils import * SEABORN = False try: @@ -59,7 +59,7 @@ GRAPHICAL_ACCESS = True try: from ete3 import TreeStyle, NodeStyle, faces, AttrFace, TextFace, CircleFace -except ImportError, e: +except ImportError as e: GRAPHICAL_ACCESS = False # define array to contains temp values @@ -118,7 +118,7 @@ def __init__(self, infile, dnafile, settings, gap_thresh, has_stop=True, logging.debug('DNA sequence was read') self.genes = set(self.dnasequences.keys()).intersection( - self.sequences.keys()) + list(self.sequences.keys())) common_spec_per_gene = {} common_spec_per_gene_len = {} @@ -145,7 +145,7 @@ def __init__(self, infile, dnafile, settings, gap_thresh, has_stop=True, "Can't find genes present in protein and nucleotide file.") logging.debug('List of selected genes : %s ' % str(self.genes)) - self.true_spec_list = set().union(*self.common_spec_per_gene.values()) + self.true_spec_list = set().union(*list(self.common_spec_per_gene.values())) if (stop_removed and has_stop): self.clean_stop() logging.debug('Stop codon removed for sequences') @@ -304,7 +304,7 @@ def accessQuality(alignfile, minqual, greater=True): return sum(highpos) cur_dir = [] - for i in xrange(loop): + for i in range(loop): d = os.path.join(outdir, '%d' % i) purge_directory(d) cur_dir.append(d) @@ -328,7 +328,7 @@ def accessQuality(alignfile, minqual, greater=True): '... trying to run hmmbuild and hmmalign ' + str(loop) + " times!") quality = [] outlist = [] - for i in xrange(loop): + for i in range(loop): outputFile = os.path.join(cur_dir[i], "alignment.sto") tmphmmfile = os.path.join(cur_dir[i], "alignment.hmm") if hmmfile: @@ -364,7 +364,7 @@ def accessQuality(alignfile, minqual, greater=True): alignment = AlignIO.read(bestiter, format="fasta") # clean by removing anything we had if clean: - for i in xrange(loop): + for i in range(loop): d = os.path.join(outdir, '%d' % i) shutil.rmtree(d, ignore_errors=True) if file_created: @@ -412,9 +412,9 @@ def _split_at_stop(self, seqlist, is_aligned): elif stop_checked: stop_removed = False core_dict = {} - gene_pos = stopmapping.values()[0] + gene_pos = list(stopmapping.values())[0] start = 0 - for i in xrange(len(gene_pos)): + for i in range(len(gene_pos)): seq_rec_list = [] for seqrec in seqlist: s_rec = SeqRecord(seqrec.seq[start:stopmapping[seqrec.id][i]], @@ -440,12 +440,12 @@ def translate(clc, core_inst, gcode=1): except: logging.warn("Wrong genetic code, resetting it to 1") - if isinstance(core_inst, basestring): + if isinstance(core_inst, str): core_inst = CoreFile(core_inst, alphabet=generic_nucleotide) core_prot_inst = {} stop_checker = lambda x: '*' if x.upper() in codontable.stop_codons else 'X' - for gene, seqs in core_inst.items(): + for gene, seqs in list(core_inst.items()): translated = [] for s in seqs: if len(s) % 3 != 0: @@ -453,7 +453,7 @@ def translate(clc, core_inst, gcode=1): "Frame-shifting detected in %s : [%s], current version does not supported it." % (s.id, gene)) else: aa_list = [codontable.forward_table.get( - s.seq[i:i + 3].upper(), stop_checker(s.seq[i:i + 3])) for i in xrange(0, len(s), 3)] + s.seq[i:i + 3].upper(), stop_checker(s.seq[i:i + 3])) for i in range(0, len(s), 3)] trans_s = Seq("".join(aa_list), generic_protein) translated.append(SeqRecord(trans_s, id=s.id, name=s.name)) core_prot_inst[gene] = translated @@ -474,8 +474,7 @@ def __init__(self, data='', alphabet=default_codon_alphabet, if rf_table is None: seq_ungapped = self._data.replace(gap_char, "") assert len(self) % 3 == 0, "Sequence length is not divisible by 3" - self.rf_table = list(filter(lambda x: x % 3 == 0, - range(len(seq_ungapped)))) + self.rf_table = list([x for x in range(len(seq_ungapped)) if x % 3 == 0]) # check alphabet # Not use Alphabet._verify_alphabet function because it # only works for single alphabet @@ -513,7 +512,7 @@ def __init__(self, aas, alignment, consensus, codon_align_dict, dct, positions, # change aa2 is a list now self.aa1, aareas = aas - self.aa2_list = aareas.keys() + self.aa2_list = list(aareas.keys()) self.codon_alignment = codon_align_dict self.dct = dct self.back_table = init_back_table(dct) @@ -544,7 +543,7 @@ def __init__(self, aas, alignment, consensus, codon_align_dict, dct, positions, def _spec_codon_usage(self): """Check codon usage in each species""" for i, aa in enumerate(self.consensus): - for spec in self.codon_alignment.keys(): + for spec in list(self.codon_alignment.keys()): spec_codon = self.codon_alignment[spec].seq.get_codon(i) spec_aa = self.dct.forward_table.get(spec_codon, None) cur_pos = self.positions[i] @@ -590,7 +589,7 @@ def get_score(self, spec, aa_ori, aa_rea): amino_counters = self.codon_map_in_genome[spec][codon] total = 0.0 numerator = 0 - for k, v in amino_counters.items(): + for k, v in list(amino_counters.items()): if k != '-': total += v try: @@ -630,11 +629,11 @@ def get_all_aas_usage(self, specie): def get_rea_aa_codon_distribution(self, specie, aa): """Get codon distribution in potentially reassigned positions""" - return dict((k, list(v)) for k, v in self.codons_distribution[specie][aa].items()) + return dict((k, list(v)) for k, v in list(self.codons_distribution[specie][aa].items())) def get_total_rea_aa_codon_distribution(self, specie, aa): """Get total codon distribution""" - return dict((k, list(v)) for k, v in self.total_codons_distribution[specie][aa].items()) + return dict((k, list(v)) for k, v in list(self.total_codons_distribution[specie][aa].items())) class SequenceSet(object): @@ -646,7 +645,7 @@ def __init__(self, coreinstance, phylotree, table_num): self.codontable = CodonTable.unambiguous_dna_by_id[abs(table_num)] self.prot_dict, self.dna_dict, self.gene_limits = coreinstance.concat() - self.prot_align = MultipleSeqAlignment(self.prot_dict.values()) + self.prot_align = MultipleSeqAlignment(list(self.prot_dict.values())) self.seqload = coreinstance self.phylotree = phylotree self.common_genome = [] @@ -672,7 +671,7 @@ def get_genes_per_species(self): gene_in_spec = {} for spec in self.common_genome: gene_in_spec[spec] = np.sum( - [1 for gene, speclist in self.seqload.common_spec_per_gene.items() if spec in speclist]) + [1 for gene, speclist in list(self.seqload.common_spec_per_gene.items()) if spec in speclist]) return gene_in_spec def restrict_to_common(self): @@ -710,15 +709,15 @@ def restrict_to_common(self): logging.debug(common_genome) # return common_genome, dict((k, v) for k, v in dna_dict.items() if k # in common_genome), prot_dict - self.dna_dict, self.prot_dict = dict((k, v) for k, v in self.dna_dict.items( - ) if k in common_genome), dict((k, v) for k, v in self.prot_dict.items() if k in common_genome) + self.dna_dict, self.prot_dict = dict((k, v) for k, v in list(self.dna_dict.items( + )) if k in common_genome), dict((k, v) for k, v in list(self.prot_dict.items()) if k in common_genome) - for k, v in self.prot_dict.items(): + for k, v in list(self.prot_dict.items()): v.seq.alphabet = generic_protein v.id = k self.prot_align = MultipleSeqAlignment( - self.prot_dict.values(), alphabet=alpha) + list(self.prot_dict.values()), alphabet=alpha) self.core = CoreFile.split_alignment(self.prot_align, self.gene_limits) @classmethod @@ -742,7 +741,7 @@ def codon_align(self, alphabet=default_codon_alphabet, gap_char='-'): # build codon alignment and return it codon_aln = [] all_undef_codon = {} - for g in self.dna_dict.keys(): + for g in list(self.dna_dict.keys()): codon_rec, undef_c = self._get_codon_record( self.dna_dict[g], self.prot_dict[g], self.codontable, alphabet) all_undef_codon[g] = undef_c @@ -752,17 +751,17 @@ def codon_align(self, alphabet=default_codon_alphabet, gap_char='-'): codon_aln, alphabet=alphabet) if all_undef_codon: - undef_codon = set().union(*all_undef_codon.values()) + undef_codon = set().union(*list(all_undef_codon.values())) logging.debug("Total position lost due to undef codon : %d" % len(undef_codon)) undef_codon = sorted(list(undef_codon)) # R_aa_position = np.asarray(xrange(self.prot_align.get_alignment_length())) # R_aa_position = np.setxor1d(tt_filter_position, np.asarray(undef_codon)) - for gene, seqrec in self.prot_dict.items(): + for gene, seqrec in list(self.prot_dict.items()): seqrec.seq._data = seqrec.seq._data.replace('X', gap_char) # self.prot_dict[gene] = seqrec self.prot_align = MultipleSeqAlignment( - self.prot_dict.values(), alphabet=alpha) + list(self.prot_dict.values()), alphabet=alpha) # remove all the position with undef codon from the dna_dict for codseqrec in self.codon_alignment: @@ -835,9 +834,9 @@ def prot_filtering(self, id_thresh=None, gap_thresh=None, ic_thresh=None, rmcnst """Filter protein alignment""" current_alignment = self.prot_align tt_filter_position = np.asarray( - xrange(current_alignment.get_alignment_length())) + range(current_alignment.get_alignment_length())) self.position = np.asarray( - xrange(current_alignment.get_alignment_length())) + range(current_alignment.get_alignment_length())) logging.debug("Alignment length %d" % current_alignment.get_alignment_length()) @@ -861,7 +860,7 @@ def prot_filtering(self, id_thresh=None, gap_thresh=None, ic_thresh=None, rmcnst # little hack for biopython < 1.69 and > 1.65 if isinstance(ic_vector, dict): ic_vector = np.zeros(len(align_info.ic_vector)) - for (ic_i, ic_v) in align_info.ic_vector.items(): + for (ic_i, ic_v) in list(align_info.ic_vector.items()): ic_vector[ic_i] = ic_v max_val = max(ic_vector) * \ ((abs(ic_thresh) <= 1 or 0.01) * abs(ic_thresh)) @@ -928,7 +927,7 @@ def filter_align_position(clc, alignment, index_array, alphabet=generic_protein, else: edited_alignment = clc.copy_codon_alignment(alignment, codontable) index_array = sorted(index_array) - for i in xrange(len(edited_alignment)): + for i in range(len(edited_alignment)): codseq = CodonSeq('') for pos in index_array: codseq += edited_alignment[i].seq.get_codon(pos) @@ -1109,7 +1108,7 @@ def get_genomes(self, use_similarity=1): self.aa_sim_json = defaultdict(list) aa2suspect = defaultdict(Counter) aa2suspect_dist = makehash(1, list) - for (j, i) in itertools.combinations(xrange(number_seq), r=2): + for (j, i) in itertools.combinations(range(number_seq), r=2): gpaired = abs( use_similarity - self.global_paired_distance[self.seq_names[i], self.seq_names[j]]) fpaired = abs( @@ -1119,7 +1118,7 @@ def get_genomes(self, use_similarity=1): paired = fpaired if self.settings.USE_GLOBAL: paired = gpaired - for aa in self.aa_paired_distance.keys(): + for aa in list(self.aa_paired_distance.keys()): aapaired = abs( use_similarity - self.aa_paired_distance[aa][self.seq_names[i], self.seq_names[j]]) self.aa_sim_json[aa_letters_1to3[aa]].append( @@ -1147,7 +1146,7 @@ def get_genomes(self, use_similarity=1): def get_suspect_by_count(self, aa2suspect, seq_num): """Find suspected species by simple counting""" - for aa in aa2suspect.keys(): + for aa in list(aa2suspect.keys()): tmp_list = aa2suspect[aa].most_common() i = 0 while i < len(tmp_list) and tmp_list[i][1] > self.settings.FREQUENCY_THRESHOLD * seq_num: @@ -1157,9 +1156,9 @@ def get_suspect_by_count(self, aa2suspect, seq_num): def get_suspect_by_stat(self, aa2suspect_dist, seq_num, use_similarity=1, test='wilcoxon', confd=0.05): """Use a statistic test to find suspected species""" - for aa in aa2suspect_dist.keys(): + for aa in list(aa2suspect_dist.keys()): tmp_list = aa2suspect_dist[aa] - for seq, val in tmp_list.items(): + for seq, val in list(tmp_list.items()): val = np.asarray(val) rank, pval = self.get_paired_test( val[:, 0], val[:, 1], use_similarity, test) @@ -1171,16 +1170,16 @@ def get_suspect_by_clustering(self, aa2suspect_dist, number_seq, use_similarity= def get_cluster(cluster_list): """Perform kmean in order to find clusters""" - features = np.asarray(cluster_list.values()) + features = np.asarray(list(cluster_list.values())) return kmeans2(features, 2, iter=100) logging.debug('Clustering chosen, labels for each species :') - for aa in aa2suspect_dist.keys(): + for aa in list(aa2suspect_dist.keys()): aafilt2dict = self.seqset.aa_filt_prot_align[aa_letters_1to3[aa]] tmp_list = aa2suspect_dist[aa] cluster_list = {} aa_set = set([]) - for seq, val in tmp_list.items(): + for seq, val in list(tmp_list.items()): val = np.mean(val, axis=0) if (val[0] > val[1] and use_similarity) or (val[0] < val[1] and not use_similarity): aa_set.add(seq) @@ -1193,8 +1192,8 @@ def get_cluster(cluster_list): logging.debug("%s\t%s\t%d" % (elem, cluster_list[elem], labels[index])) - lab1 = set(np.asarray(cluster_list.keys())[labels == 1]) - lab2 = set(np.asarray(cluster_list.keys())[labels == 0]) + lab1 = set(np.asarray(list(cluster_list.keys()))[labels == 1]) + lab2 = set(np.asarray(list(cluster_list.keys()))[labels == 0]) size1 = aa_set.intersection(lab1) size2 = aa_set.intersection(lab2) lab = lab1 if len(size1) > len(size2) else lab2 @@ -1244,7 +1243,7 @@ def possible_aa_reassignation(self): # TODO : CHANGE THIS FUNCTION TO A MORE REALISTIC ONE aa_count_spec = self.get_aa_count_in_alignment() aa_count_cons = Counter(self.filtered_consensus) - for aa, suspect in self.suspected_species.items(): + for aa, suspect in list(self.suspected_species.items()): aa_alignment = self.seqset.aa_filt_prot_align[aa_letters_1to3[aa]] suspected_list = sorted(suspect.keys()) for spec in self.seqset.common_genome: @@ -1254,11 +1253,11 @@ def possible_aa_reassignation(self): cur_aa_c2 = aa_count_spec[spec][ cur_aa] + aa_count_cons[cur_aa] filt_size = sum([aa_count_spec[spec][x] - for x in aa_count_spec[spec].keys() if x != '-']) + for x in list(aa_count_spec[spec].keys()) if x != '-']) cons_size = sum( - [aa_count_cons[x] for x in aa_count_spec.keys() if x not in ('-', 'X')]) + [aa_count_cons[x] for x in list(aa_count_spec.keys()) if x not in ('-', 'X')]) tot = sum([spec_aa_counter[x] - for x in spec_aa_counter.keys() if x != '-']) + for x in list(spec_aa_counter.keys()) if x != '-']) prob = spec_aa_counter[cur_aa] / tot if tot > 0 else 0 if prob > 0 and cur_aa != '-' and cur_aa != aa and cur_aa not in self.settings.EXCLUDE_AA_FROM: @@ -1304,7 +1303,7 @@ def get_codon_usage(self): codons_align, _ = self.seqset.get_codon_alignment() spec_data = {} codons_align = SeqIO.to_dict(codons_align) - for (spec, codalign) in codons_align.items(): + for (spec, codalign) in list(codons_align.items()): cseq = str(codalign.seq) spec_data[spec] = Counter([cseq[x:x + 3] for x in range(0, len(cseq), 3)]) @@ -1355,7 +1354,7 @@ def _valid_state(spec, valid): OUT.write("#Reference Genetic Code : %d (%s)\n" % (gcode, gcode_descr)) OUT.write("#Exception: \n") - for rea, total_rea in predict.items(): + for rea, total_rea in list(predict.items()): OUT.write("%s\n" % rea) for spec_with_rea in total_rea: OUT.write("\t%s\t%s\n" % (spec_with_rea[0].ljust( @@ -1380,14 +1379,14 @@ def save_all(self, predictions, savecodon=False): def run_analysis(self, codon_align, fcodon_align): """ Run the filtering analysis of the current dataset in sequenceset""" - for aa1, aarea in self.aa2aa_rea.items(): + for aa1, aarea in list(self.aa2aa_rea.items()): aa_alignment = self.seqset.aa_filt_prot_align[aa_letters_1to3[aa1]] gcodon_rea = CodonReaData((aa1, aarea), self.seqset.prot_align, self.global_consensus, codon_align, self.seqset.codontable, self.seqset.position, self.seqset.gene_limits, self.settings) fcodon_rea = CodonReaData((aa1, aarea), self.seqset.filt_prot_align, self.filtered_consensus, fcodon_align, self.seqset.codontable, self.seqset.filt_position, self.seqset.gene_limits, self.settings) - for aa2, species in aarea.items(): + for aa2, species in list(aarea.items()): # logging.debug("%s to %s" % (aa2, aa1)) counts = [] t = self.seqset.phylotree.copy("newick") @@ -1409,7 +1408,7 @@ def run_analysis(self, codon_align, fcodon_align): leaf.add_features(count=0) leaf.add_features(filter_count=filt_count) leaf.add_features(lost=True) - for pos in xrange(len(self.global_consensus)): + for pos in range(len(self.global_consensus)): if self.global_consensus[pos] == aa1 and rec[pos] == aa2: leaf.count += 1 @@ -1428,7 +1427,7 @@ def run_analysis(self, codon_align, fcodon_align): reacodon, usedcodon, mcodon = greacodon, gusedcodon, gmixtecodon eprob = None - if len(reacodon.values()) == 1 or len(usedcodon.values()) == 1: + if len(list(reacodon.values())) == 1 or len(list(usedcodon.values())) == 1: eprob = self.get_expected_prob_per_species(genome, aa2, aa1, use_cost=True, use_align=True) # print("%s | %s to %s : %f"%(genome, aa2, aa1, eprob)) @@ -1501,8 +1500,8 @@ def convert_tree_to_mafft(tree, seq_order, output, scale, dist_thresh=1e-10): right_branch = node.children[1] left_branch_leaf = left_branch.get_leaf_names() right_branch_leaf = right_branch.get_leaf_names() - seqnames = [min(map(lambda x: seq_order.index(x), left_branch_leaf)) + - 1, min(map(lambda x: seq_order.index(x), right_branch_leaf)) + 1, ] + seqnames = [min([seq_order.index(x) for x in left_branch_leaf]) + + 1, min([seq_order.index(x) for x in right_branch_leaf]) + 1, ] # seqnames = [seq_order.index(left_branch_leaf.name)+1, seq_order.index(right_branch_leaf.name)+1] # seqnames = [left_branch_leaf.name, right_branch_leaf.name] branchlens = [ @@ -1511,7 +1510,7 @@ def convert_tree_to_mafft(tree, seq_order, output, scale, dist_thresh=1e-10): seqnames.reverse() branchlens.reverse() - if filter(lambda x: x > 10, branchlens): + if [x for x in branchlens if x > 10]: raise ValueError( "Your branch length cannot be greater than 10.0. Mafft won't run.") @@ -1526,7 +1525,7 @@ def check_stop(seq_list, is_aligned, stop='*'): last_pos = len(seq.seq) - 1 result_map[seq.id] = [pos for pos, char in enumerate(seq.seq) if char == stop] - result = result_map.values() + result = list(result_map.values()) length_list = sorted([len(x) for x in result]) length = length_list[-1] if is_aligned: @@ -1595,7 +1594,7 @@ def remove_gap_only_columns(alignfile, curformat): """Remove all gap position from a file and return a new file""" align = AlignIO.read(alignfile, curformat) align, positions = SequenceSet.clean_alignment(align, threshold=1) - for i in xrange(len(align)): + for i in range(len(align)): seqname = align._records[i].name if hmmidpattern.match(seqname): seqname = seqname.split('|')[1] @@ -1610,7 +1609,7 @@ def remove_gap_only_columns(alignfile, curformat): def independance_test(rea, ori, genome, confd=0.05, expct_prob=0.5): """Perform a Fisher's Exact test""" codon_list = set(rea.keys()) - codon_list.update(ori.keys()) + codon_list.update(list(ori.keys())) codon_list = [x for x in codon_list if not ( rea.get(x, 0) == 0 and ori.get(x, 0) == 0)] nelmt = len(codon_list) @@ -1618,7 +1617,7 @@ def independance_test(rea, ori, genome, confd=0.05, expct_prob=0.5): # in this case, we can perform a fisher test if nelmt > 1: obs = np.zeros((nelmt, 2)) - for i in xrange(nelmt): + for i in range(nelmt): obs[i, 0] = rea.get(codon_list[i], 0) obs[i, 1] = ori.get(codon_list[i], 0) try: @@ -1633,15 +1632,15 @@ def independance_test(rea, ori, genome, confd=0.05, expct_prob=0.5): # strangely, codon is used only in rea column # complete reassignment ?? # to avoid returning 0, we return the smallest positive int - elif len(rea.values()) > 0 and len(ori.values()) == 0: + elif len(list(rea.values())) > 0 and len(list(ori.values())) == 0: return True, eps # codon is used in both column - elif len(rea.values()) > 0 and len(ori.values()) > 0: + elif len(list(rea.values())) > 0 and len(list(ori.values())) > 0: # fpval = abs(rea.values()[0] - ori.values()[0]) / \ # (rea.values()[0] + ori.values()[0]) # return rea.values()[0] >= ori.values()[0], fpval / tot_size - n = rea.values()[0] + ori.values()[0] # ignoring mixcodon ?? - pval = onevalbinomtest(rea.values()[0], n, expct_prob) + n = list(rea.values())[0] + list(ori.values())[0] # ignoring mixcodon ?? + pval = onevalbinomtest(list(rea.values())[0], n, expct_prob) return pval <= confd, pval # In this case, the codon is neither in the rea column nor in the # second column, strange result @@ -1708,8 +1707,8 @@ def scoring_function(aa1, aa2, scoring_matrix): for col in columns: al1_s = 0 al2_s = 0 - for i in xrange(nspec - 1): - for j in xrange(i + 1, nspec): + for i in range(nspec - 1): + for j in range(i + 1, nspec): al1_s += scoring_function(al1[i][col], al1[j][col], scoring_matrix) al2_s += scoring_function(al2[i][col], @@ -1724,16 +1723,16 @@ def check_align_upgrade(al1, al2, scoring_method, method="wilcoxon", column_list """Check if reassignment actually improve the alignment""" # al1 is the new alignement after changing the Gcode, and it's a dict # what we are testing is mean(al1_qual) > mean(al2_qual) - if scoring_method != "identity" and isinstance(scoring_method, basestring): + if scoring_method != "identity" and isinstance(scoring_method, str): scoring_method = getattr(MatrixInfo, scoring_method) if isinstance(al1, dict): - al1 = al1.values() + al1 = list(al1.values()) if isinstance(al2, dict): - al2 = al2.values() + al2 = list(al2.values()) al_len = len(al1[0]) nspec = len(al1) if not column_list: - column_list = range(al_len) + column_list = list(range(al_len)) assert (nspec == len(al2) and al_len == len( al2[0])), "The alignment are different" al1_sim, al2_sim = compute_SP_per_col( @@ -1747,13 +1746,13 @@ def compute_ic_content(alignment): align_info = AlignInfo.SummaryInfo(alignment) else: align_info = AlignInfo.SummaryInfo( - MultipleSeqAlignment(alignment.values())) + MultipleSeqAlignment(list(alignment.values()))) align_info.information_content() ic_vector = align_info.ic_vector # biopython wrong version hack if isinstance(ic_vector, dict): ic_vector = np.zeros(len(align_info.ic_vector)) - for (ic_i, ic_v) in align_info.ic_vector.items(): + for (ic_i, ic_v) in list(align_info.ic_vector.items()): ic_vector[ic_i] = ic_v return ic_vector.tolist() @@ -1767,7 +1766,7 @@ def check_gain(codon, cible_aa, speclist, tree, codontable, codon_alignment, def extract_alignment(codon_alignment, alignment, not_wanted_species): new_cod_alignment = [] new_alignment = [] - for spec, nucseq in codon_alignment.items(): + for spec, nucseq in list(codon_alignment.items()): if spec not in not_wanted_species: new_cod_alignment.append(nucseq) new_alignment.append(alignment[spec]) @@ -1781,10 +1780,10 @@ def translate(codon_alignment, codontable, changes=None): # [prot[i:i + 3] for i in xrange(0, len(prot), 3)]))) for spec, prot in codon_alignment.items()) translated_al = {} position = set([]) - for spec, nucseq in codon_alignment.items(): + for spec, nucseq in list(codon_alignment.items()): translated = "" nucseq_len = int(len(nucseq.seq) / 3) - for i in xrange(0, nucseq_len): + for i in range(0, nucseq_len): cod = nucseq[i * 3:(i + 1) * 3].seq.tostring() if changes.get(spec, (None,))[0] != cod: # use X for amino acid when stop codon is found @@ -1961,12 +1960,12 @@ def layout(node): spec_codonused_f = gdata[node.name]['filtered']['used_codon'] # add data - faces.add_face_to_node(PPieChartFace(spec_codonrea_f.values(), pie_size, pie_size, show_label=show_n, - colors=[fitchtree.colors[k] for k in spec_codonrea_f.keys()]), + faces.add_face_to_node(PPieChartFace(list(spec_codonrea_f.values()), pie_size, pie_size, show_label=show_n, + colors=[fitchtree.colors[k] for k in list(spec_codonrea_f.keys())]), node, column=1, position="aligned") - faces.add_face_to_node(PPieChartFace(spec_codonused_f.values(), pie_size, pie_size, show_label=show_n, - colors=[fitchtree.colors[k] for k in spec_codonused_f.keys()]), + faces.add_face_to_node(PPieChartFace(list(spec_codonused_f.values()), pie_size, pie_size, show_label=show_n, + colors=[fitchtree.colors[k] for k in list(spec_codonused_f.keys())]), node, column=2, position="aligned") next_column = 3 @@ -1974,8 +1973,8 @@ def layout(node): if(settings.SHOW_MIXTE_CODONS): spec_mixtecodon_f = gdata[node.name][ 'filtered']['mixte_codon'] - faces.add_face_to_node(PPieChartFace(spec_mixtecodon_f.values(), pie_size, pie_size, show_label=show_n, - colors=[fitchtree.colors[k] for k in spec_mixtecodon_f.keys()]), + faces.add_face_to_node(PPieChartFace(list(spec_mixtecodon_f.values()), pie_size, pie_size, show_label=show_n, + colors=[fitchtree.colors[k] for k in list(spec_mixtecodon_f.keys())]), node, column=3, position="aligned") next_column = 4 @@ -1987,20 +1986,20 @@ def layout(node): faces.add_face_to_node(LineFace( pie_size, pie_size, None), node, column=next_column, position="aligned") - faces.add_face_to_node(PPieChartFace(spec_codonrea_g.values(), pie_size, pie_size, show_label=show_n, - colors=[fitchtree.colors[k] for k in spec_codonrea_g.keys()]), + faces.add_face_to_node(PPieChartFace(list(spec_codonrea_g.values()), pie_size, pie_size, show_label=show_n, + colors=[fitchtree.colors[k] for k in list(spec_codonrea_g.keys())]), node, column=next_column + 1, position="aligned") - faces.add_face_to_node(PPieChartFace(spec_codonused_g.values(), pie_size, pie_size, show_label=show_n, - colors=[fitchtree.colors[k] for k in spec_codonused_g.keys()]), + faces.add_face_to_node(PPieChartFace(list(spec_codonused_g.values()), pie_size, pie_size, show_label=show_n, + colors=[fitchtree.colors[k] for k in list(spec_codonused_g.keys())]), node, column=next_column + 2, position="aligned") next_column += 3 if(settings.SHOW_MIXTE_CODONS): spec_mixtecodon_g = gdata[node.name][ 'global']['mixte_codon'] - faces.add_face_to_node(PPieChartFace(spec_mixtecodon_g.values(), pie_size, pie_size, show_label=show_n, - colors=[fitchtree.colors[k] for k in spec_mixtecodon_g.keys()]), + faces.add_face_to_node(PPieChartFace(list(spec_mixtecodon_g.values()), pie_size, pie_size, show_label=show_n, + colors=[fitchtree.colors[k] for k in list(spec_mixtecodon_g.keys())]), node, column=next_column, position="aligned") next_column += 1 @@ -2048,7 +2047,7 @@ def layout(node): ts.title.add_face(TextFace(fitchtree.ori_aa + " --> " + fitchtree.dest_aa, fsize=14), column=0) if(fitchtree.has_codon_data()): - for cod, col in fitchtree.colors.items(): + for cod, col in list(fitchtree.colors.items()): ts.legend.add_face(CircleFace( (pie_size / 3), col), column=0) ts.legend.add_face( @@ -2180,7 +2179,7 @@ def layout(node): ind += 2 else: - for (cod, col) in codon_col.items(): + for (cod, col) in list(codon_col.items()): ts.legend.add_face(faces.RectFace(50, 25, col, col), column=0) ts.legend.add_face(TextFace(" %s " % cod, fsize=8), column=1) ts.legend.add_face(faces.RectFace(50, 25, 'black', 'black'), column=0) @@ -2205,10 +2204,10 @@ def identify_position_with_codon(fcodal, codon, spec_to_check): """Get all positions where a codon is used""" positions = [] try: - al_len = int(len(fcodal.values()[0]) / 3) + al_len = int(len(list(fcodal.values())[0]) / 3) except: al_len = fcodal.get_aln_length() - for i in xrange(al_len): + for i in range(al_len): for spec in spec_to_check: if fcodal[spec].seq.get_codon(i) == codon: positions.append(i) @@ -2218,7 +2217,7 @@ def identify_position_with_codon(fcodal, codon, spec_to_check): def violin_plot(vals, output, score, codon, cible, imformat="pdf"): """Return violin plot of SP score """ - keys = vals.keys() + keys = list(vals.keys()) data = [vals[k] for k in keys] pos = [y + 1 for y in range(len(data))] title = 'p-values({} --> {}) : {:.2e}'.format(codon, cible, score) @@ -2301,7 +2300,7 @@ def _limit_finder(codon_pos, gene_limit=genelimit, proche=settings.STARTDIST): return gene_pos, gene_break, close_to_start - for codon in true_codon_set.keys(): + for codon in list(true_codon_set.keys()): speclist = true_codon_set[codon] if len(speclist) > 0: # pos = identify_position_with_codon(codon_align, codon, speclist) @@ -2320,8 +2319,8 @@ def _limit_finder(codon_pos, gene_limit=genelimit, proche=settings.STARTDIST): # only compute this if asked rea_pos = reafinder.update_reas( codon, cible_aa, speclist, codon_align, pos, filt_position, genelimit) - for cuspec, readt in rea_pos.items(): - for k in readt.keys(): + for cuspec, readt in list(rea_pos.items()): + for k in list(readt.keys()): rea_pos_keeper[cuspec][k] = readt[k] sp, cor_sp = alsp ic, cor_ic = alic @@ -2381,7 +2380,7 @@ def pdf_format_data(ori_aa, dest_aa, gdata, prediction, codvalid, dtype, output= cd = X_labels[i, 1] pred_dict[gm][cd] = i - for codon in codon_set.keys(): + for codon in list(codon_set.keys()): genome_list = codon_set[codon] for g in genome_list: pos = pred_dict[g][codon] @@ -2409,7 +2408,7 @@ def pdf_format_data(ori_aa, dest_aa, gdata, prediction, codvalid, dtype, output= frames = [] glist = [] - for (g, dt) in pd_data.items(): + for (g, dt) in list(pd_data.items()): glist.append(g) frames.append(pd.DataFrame.from_dict(dt, orient='index')) fdata = pd.concat(frames, keys=glist) @@ -2429,7 +2428,7 @@ def print_data_to_txt(outputfile, header, X, X_label, Y, Y_prob, codon_data, cib (["clad_valid", "trans_valid"] if valid else []))) total_elm = len(Y) - for i in xrange(total_elm): + for i in range(total_elm): end_data = [str(Y[i]), str(Y_prob[i][-1])] if valid: try: @@ -2445,7 +2444,7 @@ def print_data_to_txt(outputfile, header, X, X_label, Y, Y_prob, codon_data, cib if codon_data: out.write("\n\n### Alignment improvement:\n") - for (codon, score) in codon_data.items(): + for (codon, score) in list(codon_data.items()): out.write("{}\t{}\t{:.2e}\n".format(codon, cible, score)) tmp = Y_prob[Y > 0, 1] @@ -2454,7 +2453,7 @@ def print_data_to_txt(outputfile, header, X, X_label, Y, Y_prob, codon_data, cib write_d = "\n### Prediction Prob. range for Positive :[ %.3f - %.3f ]\n" % prerange out.write("\n{}\n".format(write_d)) - if suptext and isinstance(suptext, basestring): + if suptext and isinstance(suptext, str): out.write("\n\n{}\n".format(suptext)) out.close() diff --git a/coretracker/settings/__init__.py b/coretracker/settings/__init__.py index e45cd92..f03e1af 100755 --- a/coretracker/settings/__init__.py +++ b/coretracker/settings/__init__.py @@ -11,6 +11,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from settings import Settings -import parameters +from .settings import Settings +from . import parameters __all__ = ['Settings', 'parameters'] diff --git a/coretracker/settings/settings.py b/coretracker/settings/settings.py index 4448080..a7d3768 100644 --- a/coretracker/settings/settings.py +++ b/coretracker/settings/settings.py @@ -1,4 +1,4 @@ -import parameters +from . import parameters import Bio.SubsMat.MatrixInfo as MatrixInfo AVAILABLE_MAT = MatrixInfo.available_matrices + ['identity'] @@ -104,7 +104,7 @@ def fill(self, params): self.__dict__.update(params.__dict__) def update_params(self, **kwargs): - for k, v in kwargs.items(): + for k, v in list(kwargs.items()): if k == 'MATRIX' and v in AVAILABLE_MAT: self.__dict__[k] = v if v != "identity": diff --git a/ez_setup.py b/ez_setup.py index 7fd06b3..1e6bc88 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -153,7 +153,7 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, try: from urllib.request import urlopen except ImportError: - from urllib2 import urlopen + from urllib.request import urlopen tgz_name = "setuptools-%s.tar.gz" % version url = download_base + tgz_name saveto = os.path.join(to_dir, tgz_name) diff --git a/scripts/RF/classifierdata.py b/scripts/RF/classifierdata.py index fb76699..196a2ba 100644 --- a/scripts/RF/classifierdata.py +++ b/scripts/RF/classifierdata.py @@ -59,7 +59,7 @@ def train_load(Xlab, n, genetic_code): codon = codon.upper().replace('U', 'T') if len(codon) == 3 and codontable.forward_table[codon] != aa.upper(): datadict[genome][codon][aa] = 1 - for i in xrange(n): + for i in range(n): g, codon, ori, rea = Xlab[i] gkey = [x for x in speclist if x.match(g)] if gkey: @@ -126,7 +126,7 @@ def parse_input_file(file, genetic_code): def test_and_print(Xtest, Ytest, Xlab, clf, seuil=0.5): # get result information - print("Threshold used : %f" % seuil) + print(("Threshold used : %f" % seuil)) Ypred = clf.predict_proba(Xtest) Ypred = (Ypred[:, -1] > seuil).astype(int) clf.get_stat(Xtest, Ytest) @@ -232,7 +232,7 @@ def visualize_plot(xaxis, yaxis, xlab, ylab, title, hline=None): if hline: plt.axhline(y=hline, ls='--') plt.xlabel(xlab) - plt.xticks(range(len(xaxis)), [str(a) for a in xaxis]) + plt.xticks(list(range(len(xaxis))), [str(a) for a in xaxis]) plt.ylabel(ylab) plt.title('Classification with %s' % title) plt.savefig('data/images/' + title + '.png') @@ -267,7 +267,7 @@ def continuous_classification(c, X_train, Y_train, Xlab_train, X_test, Y_test): cur_x, cur_y = shuffle(tmp_false, tmp_false_lab, random_state=12345) c.train(cur_x, cur_y) - print("%d, Accuracy = %f" % (i, c.get_score(X_test, Y_test))) + print(("%d, Accuracy = %f" % (i, c.get_score(X_test, Y_test)))) i += 1 @@ -277,7 +277,7 @@ def oneHotFeatImport(cl, outfile="importance", features_list=[], shift_pos=9): importances = cl.clf.feature_importances_ sum_imp = np.sum(importances[shift_pos:]) importances = np.hstack((importances[:shift_pos], sum_imp)) - print(np.sum(importances)) + print((np.sum(importances))) if len(features_list) > 0 and len(features_list) != len(importances): raise ValueError("Number of features does not fit!") @@ -290,11 +290,11 @@ def oneHotFeatImport(cl, outfile="importance", features_list=[], shift_pos=9): for tree in cl.clf.estimators_], axis=0) plt.figure() plt.title("Feature importances") - plt.bar(range(n_feats), importances[ + plt.bar(list(range(n_feats)), importances[ indices], width=0.5, color="b", yerr=std[indices], align="center") if len(features_list) > 0: features_list = np.asarray(features_list)[indices] - plt.xticks(range(n_feats), features_list, rotation='vertical') + plt.xticks(list(range(n_feats)), features_list, rotation='vertical') plt.xlim([-1, n_feats]) plt.margins(0.2) @@ -344,7 +344,7 @@ def oneHotFeatImport(cl, outfile="importance", features_list=[], shift_pos=9): yeast_X_train, etiquette, feats=selected_feats) yeast_X_train_1hot = onhotencode(yeast_X_train) # undersampling(X, meta_Y_train, ratio=10, lab=meta_Xlab_train, xtest=yeast_X_train, ytest=yeast_Y_train, ylab=yeast_Xlab_train, prefix="Ori_") - print '\n\n---------------------------using one hot encoding ------------------------ \n\n' + print('\n\n---------------------------using one hot encoding ------------------------ \n\n') classifier = undersampling(X_1hot, meta_Y_train, ratio=10, lab=meta_Xlab_train, xtest=yeast_X_train_1hot, ytest=yeast_Y_train, ylab=yeast_Xlab_train, prefix="OneHot_") classifier.save_model(MODELPATH % '3') @@ -353,6 +353,6 @@ def oneHotFeatImport(cl, outfile="importance", features_list=[], shift_pos=9): "Cod. count", "Sub. count", "G. len", "Telford", "N. mixte", "Codon ID"] selected_feats = [0, 2, 3, 4, 5, 6, 7, 8, 9, 11] et = [etiquette[x] for x in selected_feats] - print et + print(et) clf = Classifier.load_from_file(MODELPATH % '3') oneHotFeatImport(clf, outfile="importance", features_list=et, shift_pos=9) diff --git a/scripts/RF/rfdataparser.py b/scripts/RF/rfdataparser.py index 8799474..80c4066 100644 --- a/scripts/RF/rfdataparser.py +++ b/scripts/RF/rfdataparser.py @@ -31,7 +31,7 @@ 'Y': 'Tyr', } -aa_letters_3to1 = dict((x[1], x[0]) for x in aa_letters_1to3.items()) +aa_letters_3to1 = dict((x[1], x[0]) for x in list(aa_letters_1to3.items())) aamap = lambda x: (aa_letters_3to1[x[0]], aa_letters_3to1[x[1]]) diff --git a/scripts/gendcoder_req.py b/scripts/gendcoder_req.py index ef9b2cc..e246f22 100755 --- a/scripts/gendcoder_req.py +++ b/scripts/gendcoder_req.py @@ -27,7 +27,7 @@ def parse_gdec_out(cntnt, outfile): for a in a_list: parts += [a.text] + [child.text for child in a.getchildren()] + \ [a.tail] - predicted_aa = "".join([x.strip() for x in filter(None, parts)]) + predicted_aa = "".join([x.strip() for x in [_f for _f in parts if _f]]) sup_infos = a_parent.tail.strip().split("\n") first = sup_infos[0].split(':')[1].strip() second = sup_infos[1].split(':')[1].strip() @@ -35,9 +35,9 @@ def parse_gdec_out(cntnt, outfile): with open(outfile, 'w') as OUT: OUT.write("#%s\n" % code) OUT.write("#codon\tGenDecoder\tExpected\n") - print "File ==> ", outfile + print("File ==> ", outfile) # print "Predicted len ==> ", len(predicted_aa) - print "Predicted ==>", predicted_aa + print("Predicted ==>", predicted_aa) # print(etree.tostring(a_parent, pretty_print=True)) for i, letter in enumerate(first): codon = letter + second[i] + third[i] @@ -98,7 +98,7 @@ def parse_gdec_out(cntnt, outfile): args.outdir, specname + ".txt")) except AssertionError: - print "*** %s : failed job, retry again" % specname + print("*** %s : failed job, retry again" % specname) tryagain -= 1 # * (1 if (start%penality) else penality/2.0)) diff --git a/setup.py b/setup.py index 32f1ff0..c4a6ee2 100755 --- a/setup.py +++ b/setup.py @@ -2,8 +2,8 @@ # # setup for CoreTracker library packages -from __future__ import absolute_import -from __future__ import print_function + + import glob import os import sys @@ -110,9 +110,9 @@ def binaries_checker(): print("Some binaries where not found : \n-%s" % "\n-".join(nfound)) answer = 'n' try: - answer = raw_input("Do you want to still continue the installation (y/n) ? ") - except: answer = input("Do you want to still continue the installation (y/n) ? ") + except: + answer = eval(input("Do you want to still continue the installation (y/n) ? ")) return answer.lower().startswith('y')