import numpy as np
import cPickle
from array import array
import sys
filename1=sys.argv[1]
filename2=sys.argv[2]
filename3=sys.argv[3]
filename4=sys.argv[4]
feature_num=sys.argv[5]
output_file=sys.argv[6]
feature_num = int(feature_num)
file1 = open(filename1,'r')
count1 = len(open(filename1,'rU').readlines())

a = np.ndarray (shape=(count1,feature_num),dtype=np.float32);
b = np.arange(count1);
j=0
for line in file1:
        #line = file1.readline()
        if line.find('#') != -1: #if there is a comment at end of line, remove it
		line = line[:line.find('#')] #"slice" the portion after the # off.
	items = line.split(' ')
	items = [x for x in items if x != ' ' and x != ''] #remove any spaces that may be left
        i=0
        while(i<feature_num):
		try:
                	feature = items[i+1].split(':')
                	a[j][i] = feature[1]
                	i=i+1
		except:
			print len(items)
			print str(i)
        b[j] = items[0]
        j=j+1
#print a
#print b
c=(a,b)


file2 = open(filename2,'r')
count2 = len(open(filename2,'rU').readlines())

a2 = np.ndarray (shape=(count2,feature_num),dtype=np.float32)
b2 = np.arange(count2)
j2=0
for line2 in file2:
        #line = file.readline()
        if line.find('#') != -1: #if there is a comment at end of line, remove it
                line = line[:line.find('#')] #"slice" the portion after the # off.
        items2 = line2.split(' ')
	items2 = [x for x in items2 if x != ' ' and x != ''] #remove any spaces that may be left
        i2=0
        while(i2<feature_num):
                feature2 = items2[i2+1].split(':')
                a2[j2][i2] = feature2[1]
                i2=i2+1
        b2[j2] = items2[0]
        j2=j2+1
#print a
#print b
c2=(a2,b2)

file3 = open(filename3,'r')
count3 = len(open(filename3,'rU').readlines())

a3 = np.ndarray (shape=(count3,feature_num),dtype=np.float32);
b3 = np.arange(count3);
j3=0
for line3 in file3:
        #line = file.readline()
        if line.find('#') != -1: #if there is a comment at end of line, remove it
                line = line[:line.find('#')] #"slice" the portion after the # off.
        items3 = line3.split(' ')
	items3 = [x for x in items3 if x != ' ' and x != ''] #remove any spaces that may be left
        i3=0
        while(i3<feature_num):
                feature3 = items3[i3+1].split(':')
                a3[j3][i3] = feature3[1]
                i3=i3+1

        b3[j3] = items3[0]
        j3=j3+1
#print a
#print b
c3=(a3,b3)


file4 = open(filename4,'r')
count4 = len(open(filename4,'rU').readlines())

a4 = np.ndarray (shape=(count4,feature_num),dtype=np.float32);
b4 = np.arange(count4);
j4=0
for line4 in file4:
        #line = file.readline()
        if line.find('#') != -1: #if there is a comment at end of line, remove it
                line = line[:line.find('#')] #"slice" the portion after the # off.
        items4 = line4.split(' ')
	items4 = [x for x in items4 if x != ' ' and x!= ''] #remove any spaces that may be left
        i4=0
        while(i4<feature_num):
                feature4 = items4[i4+1].split(':')
                a4[j4][i4] = feature4[1]
                i4=i4+1

        b4[j4] = items4[0]
        j4=j4+1
#print a
#print b
c4=(a4,b4)


file5 = open(output_file,'wb')
cPickle.dump((c,c2,c3,c4),file5,protocol=cPickle.HIGHEST_PROTOCOL)
file5.close()