diff --git a/themis/main.py b/themis/main.py index d42773c..3b6ec61 100644 --- a/themis/main.py +++ b/themis/main.py @@ -52,6 +52,9 @@ examine_truth, validate_answers_with_corpus, validate_truth_with_corpus) +from themis.rnr import create_cluster, create_config, create_ranker,query_ranker,query_untrained_ranker,\ +query_trained_rnr,query_untrained_rnr + def main(): parser = argparse.ArgumentParser(description="Themis analysis toolkit, version %s" % __version__) @@ -471,6 +474,72 @@ def answer_command(subparsers): nlc_delete.add_argument("classifiers", nargs="+", help="classifier ids") nlc_delete.set_defaults(func=nlc_delete_handler) + # Manage Rnr Model: + rnr_shared_arguments = argparse.ArgumentParser(add_help=False) + rnr_shared_arguments.add_argument("url", help="Solr url") + rnr_shared_arguments.add_argument("username", help="Solr username") + rnr_shared_arguments.add_argument("password", help="Solr password") + + rnr_parser = subparsers.add_parser("rnr", + help="answer questions with RnR") + + + rnr_subparsers = rnr_parser.add_subparsers(title="Retrieve and Rank", + description="create clusters,collection,upload config.,add documents and manage RnR", help="RnR actions") + + # Create Cluster + rnr_cluster = rnr_subparsers.add_parser("create_cluster", parents=[rnr_shared_arguments], help="Create Solr Cluster") + rnr_cluster.add_argument("--cluster_name", help="cluster name") + rnr_cluster.set_defaults(func=rnr_cluster_handler) + + # Create config, + rnr_config = rnr_subparsers.add_parser("create_config", parents=[rnr_shared_arguments], description = textwrap.dedent("This command will perform following actions " + "1.Create Solr Configuration " + "2.Create Solr Collection " + "3.Convert corpus file to rnr formated json " + "4.Add documents to Solr Collection "), help='Create config,collection and add document to colection') + rnr_config.add_argument("c_id", help="cluster id") + rnr_config.add_argument("path", help="local directory path") + rnr_config.add_argument("schema_file", help="schema file") + rnr_config.add_argument("corpus_file", help="corpus file") + rnr_config.add_argument("--config_name", help="config name") + rnr_config.add_argument("--collection_name", help="collection name") + rnr_config.set_defaults(func=rnr_config_handler) + + # create and train ranker + rnr_ranker = rnr_subparsers.add_parser("create_ranker", parents=[rnr_shared_arguments],description = textwrap.dedent("This command will perform following actions " + "1.Convert truth file to Rnr required format" + "2.Create and train ranker "), help= 'Convert truth file, create and train ranker') + rnr_ranker.add_argument("c_id", help="cluster id") + rnr_ranker.add_argument("path", help="local directory path") + rnr_ranker.add_argument("truth",help="ground truth file") + rnr_ranker.add_argument("--ranker_name", help="ranker name") + rnr_ranker.add_argument("--collection_name", help="collection name") + rnr_ranker.set_defaults(func=rnr_ranker_handler) + + + # query ranker + rnr_ranker_query = rnr_subparsers.add_parser("ranker_query", parents = [rnr_shared_arguments], help= " query the ranker ") + rnr_ranker_query.add_argument("c_id", help="cluster id") + rnr_ranker_query.add_argument("ranker", help= "ranker id") + rnr_ranker_query.add_argument("question_file", help= "question to solr") + rnr_ranker_query.add_argument("--collection_name", help="collection name") + rnr_ranker_query.set_defaults(func=rnr_ranker_query_handler) + + # query sample questions for trained RnR + rnr_sample_questions_query = rnr_subparsers.add_parser("ranker_query", parents = [rnr_shared_arguments], help= " query the ranker ") + rnr_sample_questions_query.add_argument("c_id", help="cluster id") + rnr_sample_questions_query.add_argument("ranker", help= "ranker id") + rnr_sample_questions_query.add_argument("query_file", help= "sample questions file to query solr") + rnr_sample_questions_query.set_defaults(func=rnr_query_trained_rnr_handler) + + # query sample questions for untrained RnR + rnr_untrained_sample_questions_query = rnr_subparsers.add_parser("untrained_ranker_query", parents = [rnr_shared_arguments], help= " query the ranker ") + rnr_untrained_sample_questions_query.add_argument("c_id", help="cluster id") + rnr_untrained_sample_questions_query.add_argument("query_file", help= "sample questions file to query solr") + rnr_untrained_sample_questions_query.add_argument("--collection_name", help="collection name") + rnr_untrained_sample_questions_query.set_defaults(func=rnr_query_untrained_rnr_handler) + def wea_handler(args): wea_answers = get_answers_from_usage_log(args.questions, args.qa_pairs) @@ -484,6 +553,22 @@ def solr_handler(args): def nlc_train_handler(args): print(train_nlc(args.url, args.username, args.password, args.truth, args.name)) + +def rnr_cluster_handler(args): + create_cluster(args.url, args.username, args.password,args.cluster_name) + +def rnr_config_handler(args): + print(create_config(args.url, args.username, args.password,args.c_id,args.path,args.schema_file,args.corpus_file,args.config_name,args.collection_name)) + +def rnr_ranker_handler(args): + print(create_ranker(args.url, args.username, args.password,args.c_id,args.path,args.truth,args.ranker_name,args.collection_name)) + +def rnr_ranker_query_handler(args): + print(query_ranker(args.url, args.username, args.password, args.c_id, args.ranker, args.question_file,args.collection_name)) + +def rnr_query_trained_rnr_handler(args): + print(query_trained_rnr(args.url, args.username, args.password, args.c_id, args.ranker, args.query_file)) + def nlc_router_train_handler(args): print(nlc_router_train(args.url, args.username, args.password, args.oracle_out, args.path, args.all_correct)) @@ -494,6 +579,9 @@ def nlc_router_test_handler(args): res = nlc_router_test(args.url, args.username, args.password, args.collate_file, args.path) print_csv(OracleFileType.output_format(res)) + +def rnr_query_untrained_rnr_handler(args): + print(query_untrained_rnr(args.url, args.username, args.password, args.c_id, args.query_file,args.collection_name)) def nlc_use_handler(args): corpus = args.corpus.set_index(ANSWER_ID) n = NLC(args.url, args.username, args.password, args.classifier, corpus) diff --git a/themis/rnr.py b/themis/rnr.py new file mode 100644 index 0000000..da0056b --- /dev/null +++ b/themis/rnr.py @@ -0,0 +1,291 @@ +from watson_developer_cloud import RetrieveAndRankV1 as RetriveandRank +from themis import logger, to_csv, pretty_print_json +import time +import json +import csv +import pandas +import requests, os +import subprocess +import shlex +import urllib + +def create_cluster(url, username, password,cluster_name): + if not cluster_name: + cluster_name = "solr_cluster" + rnr = RetriveandRank(url=url, username=username, password=password) + cluster = rnr.create_solr_cluster(cluster_name=cluster_name) + logger.info('Creating solr cluster....') + + # waiting for cluster to be ready + end = time.time() + 600 + try: + while time.time() < end: + if(cluster_status(url,username,password,cluster) == 'READY'): + logger.info('Cluster created successfully and ready to use. Cluster Id: %s' % cluster['solr_cluster_id']) + break + time.sleep(10) + except: + logger.info('Error in cluster creation') + +def cluster_status(url,username,password,cluster): + rnr = RetriveandRank(url=url, username=username, password=password) + return rnr.get_solr_cluster_status(cluster['solr_cluster_id'])['solr_cluster_status'] + + +def create_config(url, username, password,c_id,path,schema_file,corpus_file,config_name,collection_name): + + # create config + if not config_name: + config_name = "solr_configuration" + try: + zip_file = open(os.path.join(path,schema_file),'rb') + except: + logger.info("Error in uploading zip file from: %s" % os.path.join(path,schema_file)) + exit() + + rnr = RetriveandRank(url=url, username=username, password=password) + config = rnr.create_config(c_id,config_name,zip_file) + logger.info(pretty_print_json(config['message'])) + + # create collection + if not collection_name: + collection_name = "solr_collection" + collection = rnr.create_collection(c_id,collection_name,config_name) + if collection['success']: + logger.info('Collection successfully created as: %s'%collection_name) + else: + exit() + + # convert corpus to json format + convert_corpus_to_json(os.path.join(path,corpus_file)) + logger.info('Corpus file successfully converted to json as: corpus.json') + + # add documents to collection + logger.info('Adding documents to collection...') + status = upload_corpus(url, username, password, c_id, os.path.join(path,'corpus.json'),collection_name) + if status.__contains__('Error Code'): + logger.info('Error in uploading documents to collection') + exit() + logger.info('Documents added to the collection successfully') + + + +def create_ranker(url, username, password,c_id,path,truth,ranker_name,collection_name): + + if not collection_name: + collection_name = "solr_collection" + + # modify truth file to add relevance + logger.info('Adding relevance to ground truth....') + create_truth(os.path.join(path,truth),path) + logger.info('New file generated with relevance: rnr_relevance.csv') + + # convert relevance file into rnr format file + logger.info('Converting file....') + ranker_training_file(url, username, password, c_id, collection_name, os.path.join(path,'rnr_relevance.csv'),path) + logger.info('Conversion completed successfully. New file generated : training.txt') + + + # ranker creation and training + ''' + if not ranker_name: + ranker_name = "rnr_ranker" + rnr = RetriveandRank(url=url, username=username, password=password) + ranker = rnr.create_ranker(os.path.join(path,'training.txt'),ranker_name) + logger.info('Ranker instance is created successfully') + logger.info(ranker['status_description']) + logger.info(pretty_print_json(ranker)) + + + ''' + + # Train the ranker with the training data that was generate above from the query/relevance input + logger.info('Creating and training ranker...') + cred = username + ":" + password + if not ranker_name: + ranker_name = "rnr_ranker" + + ranker_curl_cmd = 'curl -k -X POST -u %s -F training_data=@%s -F training_metadata="{\\"name\\":\\"%s\\"}" %s' % ( + cred, os.path.join(path,'training.txt'), ranker_name, url+'/v1/rankers') + + process = subprocess.Popen(shlex.split(ranker_curl_cmd), stdout=subprocess.PIPE) + response = process.communicate()[0] + #print response[14:34] + + # waiting for ranker to be ready + end = time.time() + 900 + try: + while time.time() < end: + if (ranker_status(url, username, password, response) == 'Available'): + logger.info('Ranker is created successfully and trained with Ranker Id: %s' % response[14:34]) + break + time.sleep(10) + except: + logger.info('Error in ranker creation') + +# Ranker status +def ranker_status(url, username, password, ranker): + rnr = RetriveandRank(url=url, username=username, password=password) + return rnr.get_ranker_status(ranker[14:34])['status'] + +# modify truth file to add relevance +def create_truth(truth,path): + df = pandas.read_csv(truth) + df = df[['Question', 'Answer Id']] + df['Question'] = df['Question'].str.replace(":", "") + df['Relevance'] = 4 + df.to_csv(os.path.join(path,'rnr_relevance.csv'), index = False, header = False) + +# convert truth file in rnr format from relevance file +def ranker_training_file(url,username,password,c_id,collection_name,relevance_file,path): + url = url+'/v1/'+'solr_clusters/'+c_id+'/solr/'+collection_name+'/fcselect/' + number_row = '10' + cred = username+":"+password + with open(relevance_file, 'rb') as csvfile: + add_header = 'true' + question_relevance = csv.reader(csvfile) + with open(os.path.join(path,'training.txt'), "a") as training_file: + print ('Generating training data...') + for row in question_relevance: + question = urllib.quote(row[0]) + relevance = ','.join(row[1:]) + curl_cmd = 'curl -k -s %s -u %s -d "q=%s>=%s&generateHeader=%s&rows=%s&returnRSInput=true&wt=json" "%s"' % ( + '-v', cred, question, relevance, add_header, number_row, url) + + process = subprocess.Popen(shlex.split(curl_cmd),stdout=subprocess.PIPE) + output = process.communicate()[0] + + try: + parsed_json = json.loads(output) + if 'RSInput' in parsed_json: + training_file.write(parsed_json['RSInput']) + else: + continue + except: + print ('Command:') + print (curl_cmd) + print ('Response:') + print (output) + print (question) + raise + add_header = 'false' + print ('Generating training data complete.') + +# query ranker change +def query_ranker(url, username, password,c_id, ranker_id, query,collection_name): + if not collection_name: + collection_name = "solr_collection" + print query + cred = (username, password) + resp = requests.get(url+'/v1/solr_clusters/'+c_id+'/solr/'+collection_name+'/fcselect?ranker_id='+ranker_id+'&q='+query+'&wt=json', auth=cred) + return resp.text + +# query untrained ranker change +def query_untrained_ranker(url, username, password,c_id, query,collection_name): + cred = (username, password) + resp = requests.get(url+'/v1/solr_clusters/'+c_id+'/solr/'+collection_name+'/fcselect?q='+query+'&wt=json', auth=cred) + return resp.text + +# query trained rnr +def query_trained_rnr(url, username, password,c_id, ranker_id, question): + answers = [] + with open(question, 'r') as f: + input_reader = csv.DictReader( f, delimiter=',' ) + rows = [r for r in input_reader] + print "number of sample questions: " ,len(rows) + for row in rows: + query = row['Question'].replace("#", "").replace(":","") + resp = query_ranker(url, username, password,c_id, ranker_id, query) + try: + res = json.loads(resp) + except: + print resp.text + answers.append([query,0,"Query Error"]) + continue + + if res['response']['docs']: + answers.append([query,res['response']['docs'][0]['score'],res['response']['docs'][0]['Answer'][0]]) + else: + answers.append([query, 0, "No docs returned from RnR"]) + + with open('answers.trained.rnr.csv', 'w') as f: + output_writer = csv.writer(f) + output_writer.writerow(['Question', 'Confidence', 'Answer']) + for r in answers: + output_writer.writerow((r)) + +# query untrained rnr +def query_untrained_rnr(url, username, password,c_id, question,collection_name): + if not collection_name: + collection_name = "solr_collection" + answers = [] + with open(question, 'r') as f: + input_reader = csv.DictReader( f, delimiter=',' ) + rows = [r for r in input_reader] + print "number of sample questions: " ,len(rows) + for row in rows: + query = row['Question'].replace("#", "") + resp = query_untrained_ranker(url, username, password,c_id, query,collection_name) + try: + res = json.loads(resp) + except: + print resp.text + answers.append([query,0,"Query Error"]) + continue + + if res['response']['docs']: + answers.append([query,res['response']['docs'][0]['score'],res['response']['docs'][0]['Answer'][0]]) + else: + answers.append([query, 0, "No docs returned from RnR"]) + + with open('answers.untrained.rnr.csv', 'w') as f: + output_writer = csv.writer(f) + output_writer.writerow(['Question', 'Confidence', 'Answer']) + for r in answers: + output_writer.writerow((r)) + +# convert csv to json +def convert_corpus_to_json(corpus): + df = pandas.read_csv(corpus) + df = df[['Answer Id', 'Answer']] + f = open('corpus_temp.json', 'w') + df.to_json(f, orient='records') + + with open('corpus_temp.json', 'r') as f: + data = json.load(f) + a = [] + for row in data: + temp = {"doc": row} + a.append(("add", temp)) + + out = '{%s' % ',\n'.join(['"{}": {}'.format(action, json.dumps(dictionary)) for action, dictionary in a]) + out = out + ',"commit" : { }}' + + # can be commented as not required + with open('corpus.json', 'w') as f: + f.write(out) + +# upload documents to collection +def upload_corpus(url, username, password,c_id,corpus,collection_name): + cred = (username, password) + headers = { + 'Content-Type': 'application/json', + } + data = open(corpus) + resp = requests.post(url+'/v1/solr_clusters/'+c_id+'/solr/'+collection_name+'/update', headers=headers, data=data, auth = cred) + return resp.text + +# upload test corpus +def upload_test_corpus(url, username, password,c_id,collection_name): + cred = (username, password) + resp = requests.get(url+'/v1/solr_clusters/'+c_id+'/solr/'+collection_name+'/select?q=*:*&fl=*&df=Answer', auth = cred) + return resp.text + + + + + + + + +