1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
|
# Debugging
#work_dir = "/mnt/c/Users/james/OneDrive/RPI/Spring 2024/CSCI-1100/Homeworks/HW6/hw6_files/"
work_dir = ""
stop_word = "stop.txt"
def get_stopwords():
stopwords = []
stoptxt = open(work_dir + stop_word, "r")
stop_words = stoptxt.read().split("\n")
stoptxt.close()
stop_words = [x.strip() for x in stop_words if x.strip() != ""]
for i in stop_words:
text = ""
for j in i:
if j.isalpha():
text += j.lower()
if text != "":
stopwords.append(text)
#print("Debug - Stop words:", stopwords)
return set(stopwords)
def parse(raw):
parsed = []
parsing = raw.replace("\n"," ").replace("\t"," ").replace("\r"," ").split(" ")
#print("Debug - Parssing step 1:", parsing)
parsing = [x.strip() for x in parsing if x.strip() != ""]
#print("Debug - Parssing step 2:", parsing)
for i in parsing:
text = ""
for j in i:
if j.isalpha():
text += j.lower()
if text != "":
parsed.append(text)
#print("Debug - Parssing step 3:", parsed)
parsed = [x for x in parsed if x not in get_stopwords()]
#print("Debug - Parssing step 4:", parsed)
return parsed
def get_avg_word_len(file):
#print("Debug - File:", file)
filetxt = open(work_dir + file, "r")
raw = filetxt.read()
filetxt.close()
parsed = parse(raw)
#print("Debug - Parsed:", parsed)
avg = sum([len(x) for x in parsed]) / len(parsed)
#print("Debug - Average:", avg)
return avg
def get_ratio_distinct(file):
filetxt = open(work_dir + file, "r").read()
distinct = list(set(parse(filetxt)))
total = len(parse(filetxt))
ratio = len(distinct) / total
#print("Debug - Distinct:", ratio)
return ratio
def word_length_ranking(file):
filetxt = open(work_dir + file, "r").read()
parsed = parse(filetxt)
max_length = max([len(x) for x in parsed])
#print("Debug - Max length:", max_length)
ranking = [[] for i in range(max_length + 1)]
for i in parsed:
if i not in ranking[len(i)]:
ranking[len(i)].append(i)
#print("Debug - Adding", i, "to", len(i))
for i in range(len(ranking)):
ranking[i] = sorted(ranking[i])
#print("Debug - Ranking:", ranking)
return ranking
def get_word_set_table(file):
str1 = ""
data = word_length_ranking(file)
for i in range(1, len(data)):
cache = ""
if len(data[i]) <= 6:
cache = " ".join(data[i])
else:
cache = " ".join(data[i][:3]) + " ... "
cache += " ".join(data[i][-3:])
if cache != "":
str1 += "{:4d}:{:4d}: {}\n".format(i, len(data[i]), cache)
else:
str1 += "{:4d}:{:4d}:\n".format(i, len(data[i]))
return str1.rstrip()
def get_word_pairs(file, maxsep):
filetxt = open(work_dir + file, "r").read()
parsed = parse(filetxt)
pairs = []
for i in range(len(parsed)):
for j in range(i+1, len(parsed)):
if j - i <= maxsep:
pairs.append((parsed[i], parsed[j]))
return pairs
def get_distinct_pairs(file, maxsep):
total_pairs = get_word_pairs(file, maxsep)
pairs = []
for i in total_pairs:
cache = sorted([i[0], i[1]])
pairs.append((cache[0], cache[1]))
return sorted(list(set(pairs)))
def get_word_pair_table(file, maxsep):
pairs = get_distinct_pairs(file, maxsep)
#print("Debug - Pairs:", pairs)
str1 = " "
str1 += str(len(pairs)) + " distinct pairs" + "\n"
if len(pairs) <= 10:
for i in pairs:
str1 += " {} {}\n".format(i[0], i[1])
else:
for i in pairs[:5]:
str1 += " {} {}\n".format(i[0], i[1])
str1 += " ...\n"
for i in pairs[-5:]:
str1 += " {} {}\n".format(i[0], i[1])
return str1.rstrip()
def get_jaccard_similarity(list1, list2):
setA = set(list1)
setB = set(list2)
intersection = len(setA & setB)
union = len(setA | setB)
if union == 0:
return 0.0
else:
return intersection / union
def get_word_similarity(file1, file2):
file1txt = open(work_dir + file1, "r").read()
file2txt = open(work_dir + file2, "r").read()
parsed1 = parse(file1txt)
parsed2 = parse(file2txt)
return get_jaccard_similarity(parsed1, parsed2)
def get_word_similarity_by_length(file1, file2):
word_by_length_1 = word_length_ranking(file1)
word_by_length_2 = word_length_ranking(file2)
similarity = []
for i in range(1, max(len(word_by_length_1), len(word_by_length_2))):
if i < len(word_by_length_1) and i < len(word_by_length_2):
similarity.append(get_jaccard_similarity(word_by_length_1[i], word_by_length_2[i]))
else:
similarity.append(0.0)
return similarity
def get_word_similarity_by_length_table(file1, file2):
similarity = get_word_similarity_by_length(file1, file2)
str1 = ""
for i in range(len(similarity)):
str1 += "{:4d}: {:.4f}\n".format(i+1, similarity[i])
return str1.rstrip()
def get_word_pairs_similarity(file1, file2, maxsep):
pairs1 = get_distinct_pairs(file1, maxsep)
pairs2 = get_distinct_pairs(file2, maxsep)
return get_jaccard_similarity(pairs1, pairs2)
if __name__ == "__main__":
# Debugging
#file1st = "cat_in_the_hat.txt"
#file2rd = "pulse_morning.txt"
#maxsep = 2
#s = " 01-34 can't 42weather67 puPPy, \r \t and123\n Ch73%allenge 10ho32use,.\n"
#print(parse(s))
#get_avg_word_len(file1st)
#get_ratio_distinct(file1st)
#print(word_length_ranking(file1st)[10])
#print(get_word_set_table(file1st))
# Get user input
file1st = input("Enter the first file to analyze and compare ==> ").strip()
print(file1st)
file2rd = input("Enter the second file to analyze and compare ==> ").strip()
print(file2rd)
maxsep = int(input("Enter the maximum separation between words in a pair ==> ").strip())
print(maxsep)
files = [file1st, file2rd]
for i in files:
print("\nEvaluating document", i)
print("1. Average word length: {:.2f}".format(get_avg_word_len(i)))
print("2. Ratio of distinct words to total words: {:.3f}".format(get_ratio_distinct(i)))
print("3. Word sets for document {}:\n{}".format(i, get_word_set_table(i)))
print("4. Word pairs for document {}\n{}".format(i, get_word_pair_table(i, maxsep)))
print("5. Ratio of distinct word pairs to total: {:.3f}".format(len(get_distinct_pairs(i, maxsep)) / len(get_word_pairs(i, maxsep))))
print("\nSummary comparison")
avg_word_length_ranking = []
for i in files:
length = get_avg_word_len(i)
avg_word_length_ranking.append((i, length))
avg_word_length_ranking = sorted(avg_word_length_ranking, key=lambda x: x[1], reverse=True)
print("1. {} on average uses longer words than {}".format(avg_word_length_ranking[0][0], avg_word_length_ranking[1][0]))
print("2. Overall word use similarity: {:.3f}".format(get_word_similarity(file1st, file2rd)))
print("3. Word use similarity by length:\n{}".format(get_word_similarity_by_length_table(file1st, file2rd)))
print("4. Word pair similarity: {:.4f}".format(get_word_pairs_similarity(file1st, file2rd, maxsep)))
|
James