From 986f904d87e4b19f7ddc0cfa070784b512b99eec Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Mon, 7 Dec 2020 16:00:26 -0700
Subject: [PATCH 01/52] requirements.txt

---
 .gitignore       | 2 ++
 README_local.md  | 2 ++
 requirements.txt | 9 +++++++++
 3 files changed, 13 insertions(+)
 create mode 100644 requirements.txt

diff --git a/.gitignore b/.gitignore
index 7c21aed..9f7cd3b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 *.DS_Store
 *.csv
+__pycache__
+*.pyc
diff --git a/README_local.md b/README_local.md
index 0f831f5..6e55771 100644
--- a/README_local.md
+++ b/README_local.md
@@ -11,6 +11,8 @@ This document describes how to use 5 versions of name extraction scripts for vot
   - Levenshtein `pip install python-Levenshtein`
   - NLTK `pip install nltk`
 
+- Alternatively, if you are in an environment where you can't / don't want to install Anaconda, create and activate your Python 3 virtualenv (see [pipenv and virtualenv](https://docs.python-guide.org/dev/virtualenvs/)) and run `pip install -r requirements.txt`
+
 ## Getting Started
 Find your use case below and add your input data to the appropriate place, then run the specified python script.
 All of these scripts should be run out of the directory `Projects/NLP/SMS_Annotation`  
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..d84b090
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,9 @@
+matplotlib==3.3.3
+nltk==3.5
+numpy==1.19.4
+pandas==1.1.5
+pathlib==1.0.1
+python-Levenshtein==0.12.0
+scikit-learn==0.23.2
+scipy==1.5.4
+spacy==2.3.4
\ No newline at end of file

From c67d4eb36396379883802df680b4a2cad720442b Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Mon, 7 Dec 2020 16:00:26 -0700
Subject: [PATCH 02/52] requirements.txt

---
 .gitignore       | 2 ++
 README_local.md  | 2 ++
 requirements.txt | 9 +++++++++
 3 files changed, 13 insertions(+)
 create mode 100644 requirements.txt

diff --git a/.gitignore b/.gitignore
index 7c21aed..9f7cd3b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 *.DS_Store
 *.csv
+__pycache__
+*.pyc
diff --git a/README_local.md b/README_local.md
index 0f831f5..6e55771 100644
--- a/README_local.md
+++ b/README_local.md
@@ -11,6 +11,8 @@ This document describes how to use 5 versions of name extraction scripts for vot
   - Levenshtein `pip install python-Levenshtein`
   - NLTK `pip install nltk`
 
+- Alternatively, if you are in an environment where you can't / don't want to install Anaconda, create and activate your Python 3 virtualenv (see [pipenv and virtualenv](https://docs.python-guide.org/dev/virtualenvs/)) and run `pip install -r requirements.txt`
+
 ## Getting Started
 Find your use case below and add your input data to the appropriate place, then run the specified python script.
 All of these scripts should be run out of the directory `Projects/NLP/SMS_Annotation`  
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..d84b090
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,9 @@
+matplotlib==3.3.3
+nltk==3.5
+numpy==1.19.4
+pandas==1.1.5
+pathlib==1.0.1
+python-Levenshtein==0.12.0
+scikit-learn==0.23.2
+scipy==1.5.4
+spacy==2.3.4
\ No newline at end of file

From de022eeaccba79b891aee97d156a89940c344782 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 18 Dec 2020 16:23:33 -0700
Subject: [PATCH 03/52] stub of a Flask frontend

---
 .gitignore                                    |   2 +-
 .../Code/__pycache__/utilities.cpython-36.pyc | Bin 18029 -> 0 bytes
 .../NLP/SMS_Annotation/Input_Data/.DS_Store   | Bin 6148 -> 0 bytes
 README_local.md                               |  18 +++-
 app.py                                        |  11 +++
 requirements.txt                              |   3 +-
 templates/upload_form.html                    |  82 ++++++++++++++++++
 7 files changed, 112 insertions(+), 4 deletions(-)
 delete mode 100644 Projects/NLP/SMS_Annotation/Code/__pycache__/utilities.cpython-36.pyc
 delete mode 100644 Projects/NLP/SMS_Annotation/Input_Data/.DS_Store
 create mode 100644 app.py
 create mode 100644 templates/upload_form.html

diff --git a/.gitignore b/.gitignore
index 9f7cd3b..525f974 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-*.DS_Store
+.DS_Store
 *.csv
 __pycache__
 *.pyc
diff --git a/Projects/NLP/SMS_Annotation/Code/__pycache__/utilities.cpython-36.pyc b/Projects/NLP/SMS_Annotation/Code/__pycache__/utilities.cpython-36.pyc
deleted file mode 100644
index 153d16e9785cc3e3b1c0a078d87468ed572143e5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18029
zcmch9d5|2}d0%(WeePnhSR4e&6-WXY5Clk&7AcAr2;!h1g9{i7l5lgu(d@k5-PxU;
z8NBYr9(iLc0VP0=>Ci4GjxE}zqr{Pwt71FOKZ)ZwaSrEj;y8}nNu`vy$}+1`u2dy|
z@t@@P`(DrNfu&V)N?Fvres{m)`+nc?UgOzHrTD_9KB#|T#c}??+45IH{AK*YGoIrp
z$9I&gyteDR^7MR9o+&?tXR4iE&iEOZb<*wZa?a05+-v9kg48SeB|J0j@^ZzmxK5B!
z*}JY^4N{9EDu>u;;4bb_d4ywkJ%6vH3Q1~QQpF^-Pg12MH6f{TlG?8-{sC3>4@&Mx
zl6y!}d;|M_NbON$YOfks`_zQmKcDjts{`ubJC6Ub>Z*s-;dfmB5!F)<t49!fR6VLD
zQ8uYQp^o7B3H6wI9M2=_wmPPcqsC*wp<sN@Q#Ex0smIlm>LkjaP*15-cpg<=a4>i%
z*cXfkhtz5H^juy&qt2k_G4-r^4mFM|4|Vp=rPTMRPa?mD)C6kpQ_s(3)C=lU$UC84
zRG&uOCzVz&sqaP3NfoNI>Se^9Qs>kwC_AMhbwOQ3&S~{L{#{b9%3CLb{pz!G1$9|{
z4rNcP?^9RQYideeRoB$_tLtidzMyWX*VP;IDa_(cbrZFoQ7dXj-9qe)x})A!b;O=k
zpH~gM=Q%a2no6la&8c~{pjv8CEvfc=Y0g#4s)L%}qr7RSwz~1j;&~l3qCib{JCipS
zRwl1Dqshq^CZGP)i)WsD@yrX8XP!Rubg}qyV<lSX>K7*)?Ug79iyx8*r)q9IGTrH}
z&NkZZ=_puU4m7f|3t`k~E+Lk_)`%A3T+o?sx59<k3pyL0oS8kotj)3x&2ngzHjCPX
z+N^h1v{?x7YlVjKXd!56)9SFKHZ!|%i0#&;nAL1CDx$3M1=n#<97aLU%<D!+nP9DU
z_V|25tDu9*VYGrsWTK$aWaDPL#bmD$$y0TkQCFM!In(T}gm|S6+9<jcm<6N*v(!?i
zv4RFGon|{QomS9981xo`RrH}FzGAi;hHo{V+PL+k30t%6R%hPK>Mll!x3@4bJh~mD
z8Y}Y)Of(l-ZKZ>bjmR6&69={2UB*uus)jOiSU}XD(-<jJ%dPf0hPm3+$=jOUb~|Xy
zuLS67MK>2PqL8!cp+y)OsdZu*8*CO<!db4(YHKbqVNa$u*J^Z{!CS$t6K1}p+sSKY
zyX#4;6SU?RX3^asve4}+>`J?_T03qZXWt5Moy8`cx0|rs)q&{+n$ztC-5wTf0ezUT
ziKcBc%R$MQ=L6I21SVSTVl#CRm^p6y9G5+8t(h>mgXQ7tBdj<pA~~<SD?QVlGk3aC
z5b0J=7Mck&ObI2Sv0k6;s&ykVzG%4>VRe=o%Z+x!EOrAk+X7LT<zRU>(3rysbg{DB
zGpgHZv=#E(jZUj!T6Y?qB|OxUiPp{fs#$NWn{KxUDwS|O80>pv!JwXLwcDm^)=e9;
zwX_|AkU{9%s2;o<7}bg*%zn9L7MG0f&b3MCS3$|GWtrjR99DjLeX`S74kp1D5txI|
z)rIbsB#5TdjZC=G>veU6hHcPl5NNh$hxwU~DVmv?&c-CRsiS6Q%~3P+l$kkgW=@%t
zW>9QqKE3h8?%dgI^zZ^qBM5J^%dex_pK|l;dWUvjw(pQ#f37s=GY?AWAcn15?RJhu
z2v^#QkxtieMMAUC2$}Bb?j7u<G9k96)39xt4HRLlzSn32c$Dugc71QQ8+*}u>@6c6
zEH@!E<u8qpzw`KoPa}vN12JjbyQ#hxxr?3w{HxTvUO%PMD)WwGQj4hz&Ry4}>M4@~
zkY=a+;^j_nC8}R&M2$F00O>pf2+RkaW>*0Zn&1!l<4aTcP2(4iBbdj_Zk79Z-J)~X
z`7!TydfNF#2W>O$?kW(~i@j4%$8J-XQCWhIe&F=$A$aVxMdQ;Ky3LhkphtN6)mHS`
zmD$r<`Q!98-CYctD4e=-?ez3()Ae&5EHDPx?VLW}Rl(_%sMT%(3&T^r^^b}#0jr~C
zce(eO8b`?CW}S^A?QTQW=P<AOT&o?Nda2!QHrnB55E)iaYc68!^;DdPP*L?}cqh(*
z+gFx5VJ#h}7rM(qoM(G(pkKIf?#8+LrOQ_?#%YCR+nk^tWk=QR?>x%k<PkV-*{!-`
zu0DYHN0m3g|C2WoG}yMM8I;SOK7t5T8>j8`v6qY7QO7`4!k(u3b_df_nmcr(mWne`
zW44Wfa!j+0Fz|~I1N9zwrxCKa*}BsTW3Scqi*}k!V}1o`$En4hn$?HVo!-x2xN&w~
zMVXiMauUU15rO08@z29gPa@SMh47Cny4y4`=wuomC-QCzx*SNVma@~RdHNx=)Q1q%
zQg$d0BkqrJQga$(tVi9ti~%F})STuT=|>sJz;}$f9IO#uY>Q`6E7=J<YJCJbLkj0Q
z4%uk}iO5+Ljnp`c-d$%gb=SE|rb$h0WZ!z5QZsfq2pz<!aAj6k&?L@;gmiL2)LjZX
zJJ)ce)760T)&}Q+^6y0(p;gD##}OY6CW{1rLI=Nwh#idK6I}gnN*{_+Ds|VLW==ZF
zsI<ZG`YuxnDzKn{P32U6ORAuXTT&%e-jb@Qs-!&WWkih{#`wN;KP7oka1CQy^Y$ui
zE8d-!8sivAdNIqz>Sv;q$#9IdeN!7F$J?zX;9a|WCs4IhC%8!9%=u71hWUNuPHuQJ
zr#8G(lOMS!H@uTf9^J@Hf_^8_c4l>E^~u_?<hhZad1mI!#`tm36s%HFJO0+Wr~F&B
zjm*q5lPIhlZ+4+X1-;r^PfgxJZ_jL8qM+DSRT=6y)Bv{YK+^my^agfwmO8{nZsr+L
zBM>=$>crWZ+D2+7e6p6)k7Fir28v5_A$BjvsZP76pI}~j>f)RAE0?D()~{cDRTC3x
z*|<W$)7Jy2H68Hl*tX<(C`cWDVtXDKfW)KX5gqi}jb<?23z{t;TAb^2uTdBCD>~`#
zQVS}<P6Eu$2T>iW<I5O9NH%l4F>r5n%fE4V+~O4ZWe7|I{1<}cG{L{_5=4Okk%9Ew
zrHnp6?VJ#G5Int=uVsg^-zh*S1VgGFMscRqQNh~I@#Q%_V)He;A|wYm7*5*L&m(r@
zqdhZF0ups#W_EdfW_AcUgrgzHypIH^F=1b!*Hl8oFe6naq2bNEtjY~(*j0H|FlnJ*
z^Z`CAf{Igp@RXjDUdc6C<m5VOlkMl22U%M*xy7{5eO_u+kdsHPf~2ZQ74D`MGj|Di
zBj5)R<4y{#xtii@qk=L^aHKb`6`mi*1p^|KRn{li{!e6#oS+w2LWnjiL@QLPpF&%G
zngPMgl4&u{64N^KeldjR8l5K%`lVp4*@iY)*TKAg29@J-LXqcsUDQ8=EWgm{P6ex1
zpa;k0CYuROL#xSjy5~B|&vv@!7aDpe4U}N7G&<Ar(l^jgNH8WDkS_Tz?T&hBNYbjS
z37|s~pkHA~0zZm`EIj0lKqFy`+q4N*Bn8h9wmg*qd1RzMNrADgc_auS0jW_`CA<Ah
zfe*?ihix8gdU@NXm$~WvqAA)<FJWUXOcCp5p7GPv{2!o&-RVGo3Um56D)^JOTqt7`
ztj6D(`NYiGnI~snoOx{K@tG%P-kv#e3-Xq1y-5hQl3$`$P`BG3=NfZ!t+gPGOW5Hs
z2t%m4p+DUASsmQ*_asDcZCFEyLw}k*zl4A^anAN*X@s~13pLtF1P6r(>g~pC&<<36
z%Rm;;jcA~b5W^-}kTmqmC>=^sg4R%^Pa+}PHy{W~M>v=uuc$F&Be6x9DJ;{hWw&gr
z^Ay?c)7UeSBYz6nL1FaN%o!*)!^N$6{=Ti#3hOdefO>h3ke#ge&INg>2VjS`yxWpQ
zm|6msvIwsrJzNCpHPp)lxv8PrMF3BD%2_I4(%#fZ?#sy#!R<i<7C9*JMbzCnihL`q
z2g?oE^C%o>E*B9O3((p^6j`(c=}uTjXNHaz*hW1V{YC}L-KK6ta98wJ;BtV)li0M0
zk=szqtqx3}54my80s{Vx;Wtz$Phnq`OI)Sk(W7cFv>=!9e$iSwk++yKo+;k-Xz%n(
zQQDO5x+=Tmg>NMB+>)oifqEHeN2tf8(BGJM`sKB@ZLJFX9oxgLoBfKZP^VGel2>*#
zYd~HgFAuzD+kBFjc_1$X+^-DknDS!5_H@^Eo!jj<9gL=ok(D?aQ|_0H`<|<dn2n2Z
zyLeBLR-6DBfJUr1cQK_ZcU_rTwUfq}tN4wq{clsc;N1Ej{fe~8qE!X*2BSt!nsb77
zhmpYk%P(c#bFmVby~!n`E8xApqO{spmurTSKStf0NpG1Q>;Hrru{rzkj*gGHR56#)
zr8LTZ-Be6<DaTk;V*IYxA2p*=GRhI*y_~m$oIkX^L}jVZRE4AN<PobfhV~;;cEF5q
z1-W|fxwj9z>A>Zhfn_@)t1%Ldm=Rb^W3YMmM5AU8=31De%W!<^l7QfiAzKWvK@8eS
zke3*=2L=W$_S6f344cL+ID(2Q9-_W^x{6fn?nExV2iZS@-@pbv_8^jlyBVS%l2Je8
zur^A^2TIP-+J<K)H!^QM{nV#!eMmOmeFC&8Wdfs|z+mv!Do+M3tPC#au=T49wi`<_
z@DGrm4BToG)Dbo^6jGC7X?}NK|C^hyhq{=Bdt{{-V9Z?j8?`-}NFxxRXxgHY=;xSw
zfdO|$(>k@*ShmsY!2@Yk@Nue^^yJk=Keq;0JPKD{y&XhQ#6`%2aRx4_K*vS6k6Mw~
zYVoKQfvvf9+bYyV*xFc}p<D{-H?TT<Tnqy=Zorp`+(buG)XC{uftJ6}>V^6&2lyET
zl%985!D=0EUk?5KNeWu>e6&!fgE3s_wv|7c6aXyY6o*^XO6Ejz3=*;!{+JA@E(wIN
z5yCjlJlYI)l*uGw5$_>9Bak}r=+33_CxW#|H=6LW+m-nk!L)Z5`ZkzccpZV0Pp9%O
ze-o+WP~0ZGswi@!?wD6bepZZ+2``;WyG3s-H379Qou0t6nmXVX-7M-Hc8h3VbVrd_
zcJ*86b!e94kQhpMDvJuf63B+s+Gh<A^47~@mAXJK;It@sQ1~RCH5s6}tE;e4AtO|3
zDXps}Ymz7YE9t&>yZWYMh#!fcMPJ4%^yd)3@-JPXw>N}qyw!o*1x5?q(tdfdp}Y0f
zR%fXlXrWCQhU7PYL~Q;L%Wp|g`^?QPt_oSkE;fq2!0ETTlQ*YO^SAL!6rVJfvIu`t
z8H<VKrT>LBnp_J0(HI(v%MXwcO%>;95k(2r9csPeN}{1d-$tKidzvnVeg=llCDarV
z1PHXbSrIi^h??Bm1(U(zzSPf~yvZ)*^p~OxG-Bw}Si~<yS>|H73#OpogHe=&4oq=(
zL*jWGhv>^g_&_5UuS#ahDyhU9K^{SyT47Uf5?$YN_!Z2^ay@6QUqcXA2l|m+<XTY^
z+4T)6qf@3G)WNG^V?MCFR}@A$Pg}!s!hpALqKUu9QZ?%g@&z-wWji(}KD-MD)R*C-
zq@(}8pg~A4l>^cq11WR2%3c~4OcA?R#O{^B6uOBTmztz!{?Yy#()IsD!gzWd`VOQw
zty*jnY)@_le)+ZEu-l3)`PCULd%qZ^7Skp}?L{vj4oWx02^i+sl#(gww@nsW%-g0U
z+ClO5@4zT3LWUPRi_$VZ(1^mL(i&;onp%?NrJ?;KT2puwj{4kER)6jG(awn|hkP8z
zE;;(QOp*FgQ~-f6UIdX9A=8VCCB{sCvCO(4jH4ii3d|<AUyiEOhrn^!DJy`EK>OzL
z8V{80`I(%US)b3bur~Qbn8>G%^b_H+&~oR9r4|xOTnIYwGXs<ZOH3jkC@$iR9L-ot
z5{vgLa`iO^-_L-Ar3qv4_%`ZFkg!&PT}%2IPzZ@lHy99#{Nkpe<QIk_ATB|N#`z3D
ze=YJW329oYjZ5=5s!F<x-9>$j{S?rTMGSwBH4%g?tx(!QjQaCv@xSm3=?HKNVk?8@
z2)Co4wxUNM9mfy2;Q?X+tyTP_-8SJqw4`4~Kh|cTB8nJnXjseEe4;ajv8{Quptt0S
zVU?fSco94KoqOM`EmnXTWz8{YGvXrou9#^L*ub9t0nA>XV^AIjw4(GJg2T|+5|=XV
z&~Z43kUNx4<lrF}_mBXvjzx*B0&4f*KZ3jh_kyqKPSZHlhb=Qmr}cGk=CJ&LbI^fJ
zvL*eJqv)Bn^)s*`=2Vs^1GIjGi*i%?vv{w?*0lIi;aas~j06HPmZrN6E^Hi4Qjv<&
zi(MFCY29MG?_)5_Y*gs&TmrJjV_X93p-H<{;2Oj_<kBSYjWJ=LZ=oLkzInDuqT(NM
zj&W+VAwA#soO>=V2|y&h14jyllxXx>SVEA=5EB^^ok-##m9jCV++KF_I0Dgt4+8?R
zwD(01p7Rbm_Z>*3wDE@8u&fAo6Fv{oh8GbR5e?^LVW@V++4VZuUfEg8o>RAfkrSqx
zj`O8siHcO;!nV8V_8?liEuhINzUuRg=i<?!J~sf621{=<5GWzZNJG0BGGtFnu2kZf
z;=Q|!;W>_xl;>m*xH`nM$%Wz{ac-y@{45eYGdv*Se%QC~LrbDi#+pOyzn|`Bd9u)g
zyjkm+EF{bV<jt&ze;)!quQH%WSjxcjj42#+AYUN<@hsyEA`hvs2rz(5c**JHVQEXu
zlvDxk;ulQGN;J^@3#M#J2OV2}`-L~1+ozy;<*?sHOTpNgG&bW7bXm+7<T*q}>xt4G
zOeIc%!s9&P5(k6bPVCmDno$2Gt5CQ?<h6>HJ)dWz3h;OnspBk?pmlkoJHLe2!tqsC
zt!4!L#kG?a9ax7Hv*5K!vZTT=MTqt*gP}<M9j)h9+HJjx5%|MKWW^m!79Y0#^UO#$
zU@Gw3h~t$hzQT<r={r!R%V`g1aX5`j=Tks>@E^{(JgClIJ|*DPM?JxVCP|5Zq#&|@
z@Zs+wVp8|vzF)-tnKWat)Oqd*i@Q%v8Bg*eOH39zMg@Kq?>^3)Oy$0J*NZZXSyMo&
z%6xhnwv^<+<KBSBllSDw<yQMf+d&PuW#7r~Qlqd~gt#eNZ%vc}V!)>(b;ZwP`$^iB
zRPOz9AN~N)P=0EN8D!icW>75?%$VrAm;`|tlfCQShoWTBLN)`ZrzseZazZCwvBZjw
zQ`ig;W~m7?U?mg^i7Vl97dK(*Hg5-Z^tWKg$<2Mm#TdoIF}u0D0my^4_jf)C{ezh9
z1C!7{gj`JM3oQJz3`iSt<SQP5AKWtJjJU41ttCNb2mV?Y3Q#u#)P=f=n9JDK74>*h
zid#r=3ikP6Io$w8f0;x+AnzSx;gzWbdTZslY+1ot%kkJIm=0R=Ok&gS=B0*t0D5;e
zE{Ogl(NeRkd)PZkRNJ_QWLeu5+Urf0d5d!WN#cSH@DKRJ3ixE>6kb^?3vI+X=%>6>
z7N;4p$|YAm_BuTreRbz~T)gZTXe{#5Q@E*>`f2o2uZ`+2atrRUQO@2<3gZ+`?X=+4
z2wu8vV&&wjnh>CDBt(lPNtuwqRK>0qE{oxwjh3CpiH@aJw1<K^02kjLz*u7RD|R}G
zQmSd$Crn{UMusYMBhu)=$5FN7-c6#lQ!WCRis>wnDoc%2^wJ~l5xDOUK!ea<L7k0>
zfd_167N>kh&ixSCD9jk#{=q$<Qy=D0uSto_KPodMTmiDLgUcaW04&y6`f}A8Nc}y$
zJOOv%9>UI-p>iz(?hy0tRRARrcz_)#>yO=YkOQS-kn`otJ>EmiGUvW1CtuFpOCu+@
zB`1rcL-e1N7w(l&Uf5Eehud=xjsV^M-h1vn?_TO&=3e$*?q2?0;a(LtDT}NdWjQ8M
zf5rvkO3XFmlCuv6-^ALlz{VbDom*dnwzI!~0A|;L`|kam8M~iR<##jvgNAO~o47WT
zHwRVapfis?`-hl@W^^D)Ka`YLll0-JU=Epw%;7ou0Um}TSu_s=Rv!i2A5?oph>zJA
zoVo|hezVu?!F%&?s=xz)bFgCn5%UNPyMr|B%*0~FJR&CEAHu||(jnV_G#a@Ne?Wf{
zPp-r#;ItjX^9Z~K1=PX3%%g^-FbJd3#gThOtRgNMU=1d?1}Oi8Il?+nTaFxX)cE_w
z{$uveE$`c^L%eG{pYK1u_CD5WLe}X$tkV<yqvj~a_83RBwqYK}`sw~Lb4)!91?ISU
zLXD!g5xAs1{W-X{kDB9{@wl0=C0K)F_(?9>Jc2egmh9)u@D@1LJ^kmTU2U`7ekkop
zJLJCS(bz>u$I?DCA<pSZYwA9LtJ9!oQLRNrRcUb%Pw#F=(gzX!RkX9nxgE;CvI(CX
zN9hf<kdWvvyE>(}jcF)yj^G6{rbE^Wyw7prfFU^yDGW62Dd=fbPOM8-&XcHqCU|c(
zZWn+V^yD0Q`)Ge=bF}|oMgd)Kmr?xew;RRb!CH)Ou7y=lHdi4z{X6;K+^Ycm<SccU
zm;c;%_vIsrgPL0BE^q(fW8O}Y{{U&)Lx*y55%4=Xa`^d=dCid8{@+%3bgQV`WtM+s
zm)BE>&^i^xCPGAry`31AaBduQboxH5(u0sW8m?5a-^I+l4@D=*1xwrqQz&;oOi<?H
z3B?j#=R8`&%ChzD<6g-@Cu|$<ejc8(p`rxiR18&0HZl^8^G$AoV$k5BMJKUgW4C4b
zsbH0asyL?Lx=&8;_Y5v<;EHuu%R2lSbQl+@!PvVTnrFKDCz(qZ)j%nY_igu{TGR-4
zsAj*wWuzsMb+R6uO<vJ^1ix*ZJJc1&QBRf>_ebs0Y|#y^c9EP@eHAr9Ow@aJyzrMd
z_YI396=eJytjekRMGJ4V8V7OOj-pD-()wfVV6D~c!kmJc4_gj4BRi76-ABIjoXvU{
z2Ar;c7)`g1ANO|Wpwm{Z8+mY^vtuUT+A<S-%mPG+t_Su?SP6c@>x27YTigYS&Rc|I
zGue)~#QjPVK8|{<;yc)^Sqv;L65?c0Lj5{2{IXoxA7p)rgTK$<%M4^Lf1a@)Veq32
z{sIC&!{xezhgjm?=6d3Qj`wl&6$Z~TSZ1(<z)$m{ul0V=3Wb{mpFD)P?7ghXZE+>p
z0#U>pq65a+l^zeftQ%Iy|0merk0Yp^&`;wraG$<r9sBXP_*KP1zJtrdT@9;Q$0Op4
zx6`c8c31UZW;b78@KX$a8bLe?lIPNcHIlPD>e=jXkfG@<(LclB=NbGQgAW+|H3na2
z@RtzyWi%X~1M0uXtb?1@NkWy00Uv73@-bYINCr3%$D;p>CE)MDz{K3I4+{P4O}h)Z
zo3<G<xF3GuCZ|}}-BrK3UB3G;m~fyu*zUi{@%}9amKJb%nzykNKSx6?;^+{`!5cMM
zQU4+bn@jRAE)c??M&jb&Ahu;$ei*T<Cw%%>*^{i<2aJ(f^)E0O0+9aeOv;MND$9Dy
z%72XoVrh&$sKI=RoJ0zC_EuqIRAF22%!U7W^SYQiS{V~xxb>4*Ur84B2mj$<5`L1+
zlFe8aK6CoW<)6oV+Aa(y<T;uiPmQIDDfn6VPH!A_@iuSL(|-?R@zZZyy#9)xp1S(_
zl%KhF{pz(TKXvZHH3n}m_?(|cE;27%yn*Ps>o+W;(UE00RbpK1;a#^JI;Mm?k6-vX
z1UQp}69i{+bpOJor{Iq1r`Jx%G3XIE^)e<6jWvt=nJGAUa1sbT8ZKj`voATf55wh_
zg-)8>#wMeS+YG*sTym)^&cZrg@A0~Gq-#09V4<Y0=KOM_*$jF-DaMd}87(f*!)Qf_
z8$~O;@ny56gIot^-YBv_SmwKAOGg_@dKl18Sb!lHvfP0~tBmc;yN|f~A0Yj$pZ+QS
z{r=f+vD9(?=<J7o1RxnYwg}X0DC_eGy1|DLaBsq4&Sxrlrt!=mjTL~K(ouP6=<eDd
z<Hk3>X}Ey+t#3l;m+1sE6$SSuY$0463VoZBi$7~pRveZ+hccvW+3n}(*qm_Wg3o)A
zhr5F13H0JRbf&RKgs{%U<ti2wj=}PHXSF{fHPF+8HB?oNyjwu6(f%H|+F*h7$6#5)
zio$zvS?Bh1;)okDd-mg9{k^ab_wosAEsed;FBT#ky!FRHDJ9xl{e5Pi!ic}<sZn~&
zO<^g$Bd56JyzDHN?{X%*-t#st6FIQ%;O3pfs=@-=)&oX@{?x=5L2Ig{_Iwe&_9x7Q
zqLbIQgDVj2(zKkCD?`7E^3h>Az8M^rkI6fK59MRS@-)k_Hhb}gaX6u6HQ~axajY9w
zWj}4pWc~KRS}mx3?_$4Y+<CUY{aixx`=<y^@)yg=un3}Ow#3hDi9dTOt}Hh?D?n2)
zC2nvCuf!F~dFoSgZz3+r<!yVtq9Ab!7?GD7fX*)bKlpHOeiB%XQ_O)ZOAe`TK<)Wu
zjK?a0<ZwB)NiK~KWzq>0)G{JO?8)Qy(}NHH5B_SSalS>TPj@vG2_0vk1$H7qC#zSE
zp_{mH0bf~M;#U^JVsZ8kzSIdq{r6djyZYTmr2h^FLNQENRujC9&ns|55+bK|z<xLK
zcUk3^SfykIkci(nf64mE!(GL1>B`kNFJ7;|0*t&6mtMbiZ8ITbt;;8IB81`;h1-a%
z>v^a&aw7)ICmDm6__VBWS$SAoW_nP{+dYGl*y@S!sdZQ5aw1M8B_}bQMDN)|XyPSs
z+HYzE0$rQ<HAei}E_BC<U~gbCoI)Z%fMFBvVf^!OR8~&$Ix~;T?31xP^1T!UP6`LM
zsWT8gM-u@CA?8gcqK5)#!qYK&9)ceA>LKX;5`^Km13iG<4{rfGvjyyO0%#Br;YlzM
zQUYi<Zl6p5EsxVi+=Il$=&gW;dZ<m%;u%r`XaIn+z!gEw-Vw|LrariR(!vw~XcQ1N
z%BLCK22pziqUbgnGh+!vRfZ6?J!cc5@&XrST+_C!UKJ>*Zto$1s4;=4yc!XR0z@jD
zUhRM=fTyZRp9<xKYuoPtqF8QkJ-&nTO$ZZ+0tDgSrK|~|h$~beB)$t#ya}MUPbNUL
zcN2*8uVX*;KW6Y73?5_fPZ7j<d<qxv{=mn93C<+C9emFm5s?Tbzrwa+PyQ3ehImrT
zek^v_lgEF7>hMVaOSTg2{<j#TUbBfD`d=|gv8?|!0|JKrHw=bwq5mzDzs*3fVtfZy
z=zqtef6rhS?Ogv33;qLx-(~Ps2LF-4?;*g)c>`$qPt5xU1Hr-HXKV+PPCUq@RP%i@
zDi_Gf4;Ynfx|K=6oWh#8GqEdc4m3k*hhWYBioS+d0jR4D^)-6J;4uRkDVY8sZfIfj
zOe@@`n5Tq>pLyJ|2m4@v6#pNCIp}M=qb0{#`ah$Go#gR1m?vAzrH4`cx0xXul46Xz
z0#UIakGM+1W{Zhbyg3B581cb+#_r9X%Xi3@@i-DBhA);wx!K4i{7Eih9^E@$H8qhM
z^YkAe?icaV+rWk`b@08&Faby&CaV0r0M_83aIe7m_RJkK_V;euzr*4Cqgyhi4}WA!
z4yC~GEWC)}(a&=pKe?{BZBsIEj&qBkx?t&X3qev)Xpn-kC^&^f18h_t#|sJ8VYA@W
zo=c++uD@ZkaF6MqqkH|E4E`B|Z?{E7yG45kOR;sIMrB4@b4EC$EgATqFQ|SDw@`l*
z@AP3N+O?qFhz9z<aU}np!A|zreSg2oy=7G=UxjSqKae<;Z-C3eEf4R8enFBq;O6Zj
zb_q|2B0Fz*tNtP5zsBGnA$Tz3lX0%jPt@vl>+PCBEkDDXtm}Td(~g!jhJx>n!e*<t
zehT>-9~G0p<E;JIRf{r{!KU}t#aSS}vW0eQR=)Jr@3C&0U$g2{jN)70<sQE7jf<CG
zow|Dc;`wva7yaD1OP4O+yg2Qbu3ep;zBoO7`3)p0*Dqc<cjNNasp-#NzBcXWF5WzU
z<@E~}{Ynz4U%P(!>UH^q${(3J_u9q!^hMZn=WblRKJAw!W6<xg33I3QRg6Tw4Azs3
z9bs^s!LtmWXCUbNGGnhWxX9ow22%{03_j0bj=^;XL|WZv@D&CE+Je&58U&w&LuDJc
z(1B2d7-adEpu4TKgU{s9AS5vJ_9FlB?x{Spyqasj%d+2Q;T917jb-!MVy;-sr}CqS
zjaAF}%uegNd?jBQJxJ8#zw%@GTydY{77O{42p=8f51!>{+#RbrF3QeJ_<X*C>%oaF
G{(k{*BC&A*

diff --git a/Projects/NLP/SMS_Annotation/Input_Data/.DS_Store b/Projects/NLP/SMS_Annotation/Input_Data/.DS_Store
deleted file mode 100644
index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 6148
zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3
zem<@ulZcFPQ@L2!n>{z**<q8>++&mCkOWA81W14cNZ<zv;LbK1Poaz?KmsK2CSc!(
z0ynLxE!0092;Krf2c+FF_Fe*7ECH>lEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ
zLs35+`xjp>T0<F0fCPF1$Cyrb|F7^5{eNG?83~ZUUlGt@xh*qZDeu<Z%US-OSsOPv
j)R!Z4KLME7ReXlK;d!wEw5GODWMKRea10D2@KpjYNUI8I

diff --git a/README_local.md b/README_local.md
index 6e55771..430b09a 100644
--- a/README_local.md
+++ b/README_local.md
@@ -13,6 +13,8 @@ This document describes how to use 5 versions of name extraction scripts for vot
 
 - Alternatively, if you are in an environment where you can't / don't want to install Anaconda, create and activate your Python 3 virtualenv (see [pipenv and virtualenv](https://docs.python-guide.org/dev/virtualenvs/)) and run `pip install -r requirements.txt`
 
+- You'll also need to run `spacy download en` once.
+
 ## Getting Started
 Find your use case below and add your input data to the appropriate place, then run the specified python script.
 All of these scripts should be run out of the directory `Projects/NLP/SMS_Annotation`  
@@ -84,10 +86,10 @@ This script will output two files:
 Add a csv to the Input_Data folder. This csv file must contain column 'names' containing the names logged by a text banker  
 
 **Instructions:**
-In this directory, run `python3 Code/name_cleaning.py -d [input_filename]`  
+In this directory, run `python3 Code/name_cleaning.py -i [input_filename]`
 
 **Outputs:**
-A File named `labeled_names_cleaned_no_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file 
+A file in `Output_Data` named `labeled_names_cleaned_no_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file 
   
   
 ## Text Banker Log Cleaning (utilizing text message conversation)
@@ -128,3 +130,15 @@ This script will output two files:
 - *ContactName* a unique identifier for the conversation
 - *NoteText* free text possibly including names of tripling targets
 - *names_extract* a guess for the extracted names (to be reviewed)
+
+# Running the app frontend
+app.py is a Flask-based frontend that provides a dedicated UI for uploading data sets and running the scripts with them.
+
+Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
+
+To run an instance of the frontend locally:
+`
+export FLASK_APP=app.py
+export FLASK_ENV=development
+flask run
+`
\ No newline at end of file
diff --git a/app.py b/app.py
new file mode 100644
index 0000000..133ddb6
--- /dev/null
+++ b/app.py
@@ -0,0 +1,11 @@
+from flask import Flask, request, render_template
+app = Flask(__name__)
+
+@app.route('/')
+def index():
+    if request.method == 'POST':
+        import pdb; pdb.set_trace();
+        # handle file upload
+        return
+    else: 
+        return render_template('upload_form.html')
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index d84b090..fc5c90f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,5 @@ pathlib==1.0.1
 python-Levenshtein==0.12.0
 scikit-learn==0.23.2
 scipy==1.5.4
-spacy==2.3.4
\ No newline at end of file
+spacy==2.3.4
+Flask==1.1.2
\ No newline at end of file
diff --git a/templates/upload_form.html b/templates/upload_form.html
new file mode 100644
index 0000000..363344f
--- /dev/null
+++ b/templates/upload_form.html
@@ -0,0 +1,82 @@
+<html>
+
+<head>
+    <title>
+        Vote tripling SMS transcript parsing
+    </title>
+
+    <style>
+        body { width: 96%; padding: 2%; }
+        h1 h2 h3 h4 h5 h6 { font-family: 'sans-serif'; }
+        main { max-width: 800px; width: 100%; margin: auto; }
+        ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
+        ul.menu li a { padding: .5em; color: darkblue; background-color: #ccc;}
+    </style>
+</head>
+
+<body>
+
+    <main>
+
+        <h1>Vote tripling SMS transcript parsing</h1>
+
+        <ul class="menu"><li><a href="#van_export_cleaning">VAN export cleaning</a></li>
+            <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+        </ul>
+        
+
+        <hr />
+
+        <h3 id='van_export_cleaning'>VAN Export Cleaning</h3>
+         <p><strong>I have a VAN Export and I need to extract any tripling target names from the note text.</strong></p>
+
+         <p>Upload a csv file with the following columns:</p>
+         <ul>
+        <li>*VANID* a unique ID for this row
+        <li>*ContactName* the name of the tripler
+        <li>*NoteText* free text possibly including names of tripling targets
+        </ul>
+
+        <form method="post" enctype="multipart/form-data">
+            <input type="file" id="vec_file" name="vec_file" accept=".csv">
+            <button>Submit</button>
+        </form>
+
+        <p>If successful, the output will include two files: 
+        <ul><li>1. A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+            <ul><li>- *VANID* a unique identifier for the conversation
+                <li>- *names_extract* the extracted names
+            </ul>
+
+            <li>2. A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+                <ul><li>- *VANID* a unique identifier for the conversation
+                    <li>- *ContactName* a unique identifier for the conversation
+                    <li>- *NoteText* free text possibly including names of tripling targets
+                    <li>- *names_extract* a guess for the extracted names (to be reviewed)
+                </ul>
+        </ul>
+
+        <hr />
+
+        <h3 id="text_banker_log_cleaning"> Text Banker Log Cleaning (utilizing text message conversation)</h3>
+        <p><strong> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </strong></p>
+
+        <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
+
+        <p>Upload a csv file of the same format as the output of the aggregation in step 1.
+        This csv file must also contain column 'names' containing the names logged by a text banker.</p>
+
+        <form method="post" enctype="multipart/form-data">
+            <input type="file" id="tblc_file" name="tblc_file" accept=".csv">
+            <button>Submit</button>
+        </form>
+
+        <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+
+        <hr/>
+
+    </main>
+
+</body>
+
+</html>
\ No newline at end of file

From 5cb21a9fcafca245231e14452bc7075087cfe9de Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 18 Dec 2020 16:23:33 -0700
Subject: [PATCH 04/52] stub of a Flask frontend

---
 .gitignore                                    |   2 +-
 .../Code/__pycache__/utilities.cpython-36.pyc | Bin 18029 -> 0 bytes
 .../NLP/SMS_Annotation/Input_Data/.DS_Store   | Bin 6148 -> 0 bytes
 README_local.md                               |  18 +++-
 app.py                                        |  11 +++
 requirements.txt                              |   3 +-
 templates/upload_form.html                    |  82 ++++++++++++++++++
 7 files changed, 112 insertions(+), 4 deletions(-)
 delete mode 100644 Projects/NLP/SMS_Annotation/Code/__pycache__/utilities.cpython-36.pyc
 delete mode 100644 Projects/NLP/SMS_Annotation/Input_Data/.DS_Store
 create mode 100644 app.py
 create mode 100644 templates/upload_form.html

diff --git a/.gitignore b/.gitignore
index 9f7cd3b..525f974 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-*.DS_Store
+.DS_Store
 *.csv
 __pycache__
 *.pyc
diff --git a/Projects/NLP/SMS_Annotation/Code/__pycache__/utilities.cpython-36.pyc b/Projects/NLP/SMS_Annotation/Code/__pycache__/utilities.cpython-36.pyc
deleted file mode 100644
index 153d16e9785cc3e3b1c0a078d87468ed572143e5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18029
zcmch9d5|2}d0%(WeePnhSR4e&6-WXY5Clk&7AcAr2;!h1g9{i7l5lgu(d@k5-PxU;
z8NBYr9(iLc0VP0=>Ci4GjxE}zqr{Pwt71FOKZ)ZwaSrEj;y8}nNu`vy$}+1`u2dy|
z@t@@P`(DrNfu&V)N?Fvres{m)`+nc?UgOzHrTD_9KB#|T#c}??+45IH{AK*YGoIrp
z$9I&gyteDR^7MR9o+&?tXR4iE&iEOZb<*wZa?a05+-v9kg48SeB|J0j@^ZzmxK5B!
z*}JY^4N{9EDu>u;;4bb_d4ywkJ%6vH3Q1~QQpF^-Pg12MH6f{TlG?8-{sC3>4@&Mx
zl6y!}d;|M_NbON$YOfks`_zQmKcDjts{`ubJC6Ub>Z*s-;dfmB5!F)<t49!fR6VLD
zQ8uYQp^o7B3H6wI9M2=_wmPPcqsC*wp<sN@Q#Ex0smIlm>LkjaP*15-cpg<=a4>i%
z*cXfkhtz5H^juy&qt2k_G4-r^4mFM|4|Vp=rPTMRPa?mD)C6kpQ_s(3)C=lU$UC84
zRG&uOCzVz&sqaP3NfoNI>Se^9Qs>kwC_AMhbwOQ3&S~{L{#{b9%3CLb{pz!G1$9|{
z4rNcP?^9RQYideeRoB$_tLtidzMyWX*VP;IDa_(cbrZFoQ7dXj-9qe)x})A!b;O=k
zpH~gM=Q%a2no6la&8c~{pjv8CEvfc=Y0g#4s)L%}qr7RSwz~1j;&~l3qCib{JCipS
zRwl1Dqshq^CZGP)i)WsD@yrX8XP!Rubg}qyV<lSX>K7*)?Ug79iyx8*r)q9IGTrH}
z&NkZZ=_puU4m7f|3t`k~E+Lk_)`%A3T+o?sx59<k3pyL0oS8kotj)3x&2ngzHjCPX
z+N^h1v{?x7YlVjKXd!56)9SFKHZ!|%i0#&;nAL1CDx$3M1=n#<97aLU%<D!+nP9DU
z_V|25tDu9*VYGrsWTK$aWaDPL#bmD$$y0TkQCFM!In(T}gm|S6+9<jcm<6N*v(!?i
zv4RFGon|{QomS9981xo`RrH}FzGAi;hHo{V+PL+k30t%6R%hPK>Mll!x3@4bJh~mD
z8Y}Y)Of(l-ZKZ>bjmR6&69={2UB*uus)jOiSU}XD(-<jJ%dPf0hPm3+$=jOUb~|Xy
zuLS67MK>2PqL8!cp+y)OsdZu*8*CO<!db4(YHKbqVNa$u*J^Z{!CS$t6K1}p+sSKY
zyX#4;6SU?RX3^asve4}+>`J?_T03qZXWt5Moy8`cx0|rs)q&{+n$ztC-5wTf0ezUT
ziKcBc%R$MQ=L6I21SVSTVl#CRm^p6y9G5+8t(h>mgXQ7tBdj<pA~~<SD?QVlGk3aC
z5b0J=7Mck&ObI2Sv0k6;s&ykVzG%4>VRe=o%Z+x!EOrAk+X7LT<zRU>(3rysbg{DB
zGpgHZv=#E(jZUj!T6Y?qB|OxUiPp{fs#$NWn{KxUDwS|O80>pv!JwXLwcDm^)=e9;
zwX_|AkU{9%s2;o<7}bg*%zn9L7MG0f&b3MCS3$|GWtrjR99DjLeX`S74kp1D5txI|
z)rIbsB#5TdjZC=G>veU6hHcPl5NNh$hxwU~DVmv?&c-CRsiS6Q%~3P+l$kkgW=@%t
zW>9QqKE3h8?%dgI^zZ^qBM5J^%dex_pK|l;dWUvjw(pQ#f37s=GY?AWAcn15?RJhu
z2v^#QkxtieMMAUC2$}Bb?j7u<G9k96)39xt4HRLlzSn32c$Dugc71QQ8+*}u>@6c6
zEH@!E<u8qpzw`KoPa}vN12JjbyQ#hxxr?3w{HxTvUO%PMD)WwGQj4hz&Ry4}>M4@~
zkY=a+;^j_nC8}R&M2$F00O>pf2+RkaW>*0Zn&1!l<4aTcP2(4iBbdj_Zk79Z-J)~X
z`7!TydfNF#2W>O$?kW(~i@j4%$8J-XQCWhIe&F=$A$aVxMdQ;Ky3LhkphtN6)mHS`
zmD$r<`Q!98-CYctD4e=-?ez3()Ae&5EHDPx?VLW}Rl(_%sMT%(3&T^r^^b}#0jr~C
zce(eO8b`?CW}S^A?QTQW=P<AOT&o?Nda2!QHrnB55E)iaYc68!^;DdPP*L?}cqh(*
z+gFx5VJ#h}7rM(qoM(G(pkKIf?#8+LrOQ_?#%YCR+nk^tWk=QR?>x%k<PkV-*{!-`
zu0DYHN0m3g|C2WoG}yMM8I;SOK7t5T8>j8`v6qY7QO7`4!k(u3b_df_nmcr(mWne`
zW44Wfa!j+0Fz|~I1N9zwrxCKa*}BsTW3Scqi*}k!V}1o`$En4hn$?HVo!-x2xN&w~
zMVXiMauUU15rO08@z29gPa@SMh47Cny4y4`=wuomC-QCzx*SNVma@~RdHNx=)Q1q%
zQg$d0BkqrJQga$(tVi9ti~%F})STuT=|>sJz;}$f9IO#uY>Q`6E7=J<YJCJbLkj0Q
z4%uk}iO5+Ljnp`c-d$%gb=SE|rb$h0WZ!z5QZsfq2pz<!aAj6k&?L@;gmiL2)LjZX
zJJ)ce)760T)&}Q+^6y0(p;gD##}OY6CW{1rLI=Nwh#idK6I}gnN*{_+Ds|VLW==ZF
zsI<ZG`YuxnDzKn{P32U6ORAuXTT&%e-jb@Qs-!&WWkih{#`wN;KP7oka1CQy^Y$ui
zE8d-!8sivAdNIqz>Sv;q$#9IdeN!7F$J?zX;9a|WCs4IhC%8!9%=u71hWUNuPHuQJ
zr#8G(lOMS!H@uTf9^J@Hf_^8_c4l>E^~u_?<hhZad1mI!#`tm36s%HFJO0+Wr~F&B
zjm*q5lPIhlZ+4+X1-;r^PfgxJZ_jL8qM+DSRT=6y)Bv{YK+^my^agfwmO8{nZsr+L
zBM>=$>crWZ+D2+7e6p6)k7Fir28v5_A$BjvsZP76pI}~j>f)RAE0?D()~{cDRTC3x
z*|<W$)7Jy2H68Hl*tX<(C`cWDVtXDKfW)KX5gqi}jb<?23z{t;TAb^2uTdBCD>~`#
zQVS}<P6Eu$2T>iW<I5O9NH%l4F>r5n%fE4V+~O4ZWe7|I{1<}cG{L{_5=4Okk%9Ew
zrHnp6?VJ#G5Int=uVsg^-zh*S1VgGFMscRqQNh~I@#Q%_V)He;A|wYm7*5*L&m(r@
zqdhZF0ups#W_EdfW_AcUgrgzHypIH^F=1b!*Hl8oFe6naq2bNEtjY~(*j0H|FlnJ*
z^Z`CAf{Igp@RXjDUdc6C<m5VOlkMl22U%M*xy7{5eO_u+kdsHPf~2ZQ74D`MGj|Di
zBj5)R<4y{#xtii@qk=L^aHKb`6`mi*1p^|KRn{li{!e6#oS+w2LWnjiL@QLPpF&%G
zngPMgl4&u{64N^KeldjR8l5K%`lVp4*@iY)*TKAg29@J-LXqcsUDQ8=EWgm{P6ex1
zpa;k0CYuROL#xSjy5~B|&vv@!7aDpe4U}N7G&<Ar(l^jgNH8WDkS_Tz?T&hBNYbjS
z37|s~pkHA~0zZm`EIj0lKqFy`+q4N*Bn8h9wmg*qd1RzMNrADgc_auS0jW_`CA<Ah
zfe*?ihix8gdU@NXm$~WvqAA)<FJWUXOcCp5p7GPv{2!o&-RVGo3Um56D)^JOTqt7`
ztj6D(`NYiGnI~snoOx{K@tG%P-kv#e3-Xq1y-5hQl3$`$P`BG3=NfZ!t+gPGOW5Hs
z2t%m4p+DUASsmQ*_asDcZCFEyLw}k*zl4A^anAN*X@s~13pLtF1P6r(>g~pC&<<36
z%Rm;;jcA~b5W^-}kTmqmC>=^sg4R%^Pa+}PHy{W~M>v=uuc$F&Be6x9DJ;{hWw&gr
z^Ay?c)7UeSBYz6nL1FaN%o!*)!^N$6{=Ti#3hOdefO>h3ke#ge&INg>2VjS`yxWpQ
zm|6msvIwsrJzNCpHPp)lxv8PrMF3BD%2_I4(%#fZ?#sy#!R<i<7C9*JMbzCnihL`q
z2g?oE^C%o>E*B9O3((p^6j`(c=}uTjXNHaz*hW1V{YC}L-KK6ta98wJ;BtV)li0M0
zk=szqtqx3}54my80s{Vx;Wtz$Phnq`OI)Sk(W7cFv>=!9e$iSwk++yKo+;k-Xz%n(
zQQDO5x+=Tmg>NMB+>)oifqEHeN2tf8(BGJM`sKB@ZLJFX9oxgLoBfKZP^VGel2>*#
zYd~HgFAuzD+kBFjc_1$X+^-DknDS!5_H@^Eo!jj<9gL=ok(D?aQ|_0H`<|<dn2n2Z
zyLeBLR-6DBfJUr1cQK_ZcU_rTwUfq}tN4wq{clsc;N1Ej{fe~8qE!X*2BSt!nsb77
zhmpYk%P(c#bFmVby~!n`E8xApqO{spmurTSKStf0NpG1Q>;Hrru{rzkj*gGHR56#)
zr8LTZ-Be6<DaTk;V*IYxA2p*=GRhI*y_~m$oIkX^L}jVZRE4AN<PobfhV~;;cEF5q
z1-W|fxwj9z>A>Zhfn_@)t1%Ldm=Rb^W3YMmM5AU8=31De%W!<^l7QfiAzKWvK@8eS
zke3*=2L=W$_S6f344cL+ID(2Q9-_W^x{6fn?nExV2iZS@-@pbv_8^jlyBVS%l2Je8
zur^A^2TIP-+J<K)H!^QM{nV#!eMmOmeFC&8Wdfs|z+mv!Do+M3tPC#au=T49wi`<_
z@DGrm4BToG)Dbo^6jGC7X?}NK|C^hyhq{=Bdt{{-V9Z?j8?`-}NFxxRXxgHY=;xSw
zfdO|$(>k@*ShmsY!2@Yk@Nue^^yJk=Keq;0JPKD{y&XhQ#6`%2aRx4_K*vS6k6Mw~
zYVoKQfvvf9+bYyV*xFc}p<D{-H?TT<Tnqy=Zorp`+(buG)XC{uftJ6}>V^6&2lyET
zl%985!D=0EUk?5KNeWu>e6&!fgE3s_wv|7c6aXyY6o*^XO6Ejz3=*;!{+JA@E(wIN
z5yCjlJlYI)l*uGw5$_>9Bak}r=+33_CxW#|H=6LW+m-nk!L)Z5`ZkzccpZV0Pp9%O
ze-o+WP~0ZGswi@!?wD6bepZZ+2``;WyG3s-H379Qou0t6nmXVX-7M-Hc8h3VbVrd_
zcJ*86b!e94kQhpMDvJuf63B+s+Gh<A^47~@mAXJK;It@sQ1~RCH5s6}tE;e4AtO|3
zDXps}Ymz7YE9t&>yZWYMh#!fcMPJ4%^yd)3@-JPXw>N}qyw!o*1x5?q(tdfdp}Y0f
zR%fXlXrWCQhU7PYL~Q;L%Wp|g`^?QPt_oSkE;fq2!0ETTlQ*YO^SAL!6rVJfvIu`t
z8H<VKrT>LBnp_J0(HI(v%MXwcO%>;95k(2r9csPeN}{1d-$tKidzvnVeg=llCDarV
z1PHXbSrIi^h??Bm1(U(zzSPf~yvZ)*^p~OxG-Bw}Si~<yS>|H73#OpogHe=&4oq=(
zL*jWGhv>^g_&_5UuS#ahDyhU9K^{SyT47Uf5?$YN_!Z2^ay@6QUqcXA2l|m+<XTY^
z+4T)6qf@3G)WNG^V?MCFR}@A$Pg}!s!hpALqKUu9QZ?%g@&z-wWji(}KD-MD)R*C-
zq@(}8pg~A4l>^cq11WR2%3c~4OcA?R#O{^B6uOBTmztz!{?Yy#()IsD!gzWd`VOQw
zty*jnY)@_le)+ZEu-l3)`PCULd%qZ^7Skp}?L{vj4oWx02^i+sl#(gww@nsW%-g0U
z+ClO5@4zT3LWUPRi_$VZ(1^mL(i&;onp%?NrJ?;KT2puwj{4kER)6jG(awn|hkP8z
zE;;(QOp*FgQ~-f6UIdX9A=8VCCB{sCvCO(4jH4ii3d|<AUyiEOhrn^!DJy`EK>OzL
z8V{80`I(%US)b3bur~Qbn8>G%^b_H+&~oR9r4|xOTnIYwGXs<ZOH3jkC@$iR9L-ot
z5{vgLa`iO^-_L-Ar3qv4_%`ZFkg!&PT}%2IPzZ@lHy99#{Nkpe<QIk_ATB|N#`z3D
ze=YJW329oYjZ5=5s!F<x-9>$j{S?rTMGSwBH4%g?tx(!QjQaCv@xSm3=?HKNVk?8@
z2)Co4wxUNM9mfy2;Q?X+tyTP_-8SJqw4`4~Kh|cTB8nJnXjseEe4;ajv8{Quptt0S
zVU?fSco94KoqOM`EmnXTWz8{YGvXrou9#^L*ub9t0nA>XV^AIjw4(GJg2T|+5|=XV
z&~Z43kUNx4<lrF}_mBXvjzx*B0&4f*KZ3jh_kyqKPSZHlhb=Qmr}cGk=CJ&LbI^fJ
zvL*eJqv)Bn^)s*`=2Vs^1GIjGi*i%?vv{w?*0lIi;aas~j06HPmZrN6E^Hi4Qjv<&
zi(MFCY29MG?_)5_Y*gs&TmrJjV_X93p-H<{;2Oj_<kBSYjWJ=LZ=oLkzInDuqT(NM
zj&W+VAwA#soO>=V2|y&h14jyllxXx>SVEA=5EB^^ok-##m9jCV++KF_I0Dgt4+8?R
zwD(01p7Rbm_Z>*3wDE@8u&fAo6Fv{oh8GbR5e?^LVW@V++4VZuUfEg8o>RAfkrSqx
zj`O8siHcO;!nV8V_8?liEuhINzUuRg=i<?!J~sf621{=<5GWzZNJG0BGGtFnu2kZf
z;=Q|!;W>_xl;>m*xH`nM$%Wz{ac-y@{45eYGdv*Se%QC~LrbDi#+pOyzn|`Bd9u)g
zyjkm+EF{bV<jt&ze;)!quQH%WSjxcjj42#+AYUN<@hsyEA`hvs2rz(5c**JHVQEXu
zlvDxk;ulQGN;J^@3#M#J2OV2}`-L~1+ozy;<*?sHOTpNgG&bW7bXm+7<T*q}>xt4G
zOeIc%!s9&P5(k6bPVCmDno$2Gt5CQ?<h6>HJ)dWz3h;OnspBk?pmlkoJHLe2!tqsC
zt!4!L#kG?a9ax7Hv*5K!vZTT=MTqt*gP}<M9j)h9+HJjx5%|MKWW^m!79Y0#^UO#$
zU@Gw3h~t$hzQT<r={r!R%V`g1aX5`j=Tks>@E^{(JgClIJ|*DPM?JxVCP|5Zq#&|@
z@Zs+wVp8|vzF)-tnKWat)Oqd*i@Q%v8Bg*eOH39zMg@Kq?>^3)Oy$0J*NZZXSyMo&
z%6xhnwv^<+<KBSBllSDw<yQMf+d&PuW#7r~Qlqd~gt#eNZ%vc}V!)>(b;ZwP`$^iB
zRPOz9AN~N)P=0EN8D!icW>75?%$VrAm;`|tlfCQShoWTBLN)`ZrzseZazZCwvBZjw
zQ`ig;W~m7?U?mg^i7Vl97dK(*Hg5-Z^tWKg$<2Mm#TdoIF}u0D0my^4_jf)C{ezh9
z1C!7{gj`JM3oQJz3`iSt<SQP5AKWtJjJU41ttCNb2mV?Y3Q#u#)P=f=n9JDK74>*h
zid#r=3ikP6Io$w8f0;x+AnzSx;gzWbdTZslY+1ot%kkJIm=0R=Ok&gS=B0*t0D5;e
zE{Ogl(NeRkd)PZkRNJ_QWLeu5+Urf0d5d!WN#cSH@DKRJ3ixE>6kb^?3vI+X=%>6>
z7N;4p$|YAm_BuTreRbz~T)gZTXe{#5Q@E*>`f2o2uZ`+2atrRUQO@2<3gZ+`?X=+4
z2wu8vV&&wjnh>CDBt(lPNtuwqRK>0qE{oxwjh3CpiH@aJw1<K^02kjLz*u7RD|R}G
zQmSd$Crn{UMusYMBhu)=$5FN7-c6#lQ!WCRis>wnDoc%2^wJ~l5xDOUK!ea<L7k0>
zfd_167N>kh&ixSCD9jk#{=q$<Qy=D0uSto_KPodMTmiDLgUcaW04&y6`f}A8Nc}y$
zJOOv%9>UI-p>iz(?hy0tRRARrcz_)#>yO=YkOQS-kn`otJ>EmiGUvW1CtuFpOCu+@
zB`1rcL-e1N7w(l&Uf5Eehud=xjsV^M-h1vn?_TO&=3e$*?q2?0;a(LtDT}NdWjQ8M
zf5rvkO3XFmlCuv6-^ALlz{VbDom*dnwzI!~0A|;L`|kam8M~iR<##jvgNAO~o47WT
zHwRVapfis?`-hl@W^^D)Ka`YLll0-JU=Epw%;7ou0Um}TSu_s=Rv!i2A5?oph>zJA
zoVo|hezVu?!F%&?s=xz)bFgCn5%UNPyMr|B%*0~FJR&CEAHu||(jnV_G#a@Ne?Wf{
zPp-r#;ItjX^9Z~K1=PX3%%g^-FbJd3#gThOtRgNMU=1d?1}Oi8Il?+nTaFxX)cE_w
z{$uveE$`c^L%eG{pYK1u_CD5WLe}X$tkV<yqvj~a_83RBwqYK}`sw~Lb4)!91?ISU
zLXD!g5xAs1{W-X{kDB9{@wl0=C0K)F_(?9>Jc2egmh9)u@D@1LJ^kmTU2U`7ekkop
zJLJCS(bz>u$I?DCA<pSZYwA9LtJ9!oQLRNrRcUb%Pw#F=(gzX!RkX9nxgE;CvI(CX
zN9hf<kdWvvyE>(}jcF)yj^G6{rbE^Wyw7prfFU^yDGW62Dd=fbPOM8-&XcHqCU|c(
zZWn+V^yD0Q`)Ge=bF}|oMgd)Kmr?xew;RRb!CH)Ou7y=lHdi4z{X6;K+^Ycm<SccU
zm;c;%_vIsrgPL0BE^q(fW8O}Y{{U&)Lx*y55%4=Xa`^d=dCid8{@+%3bgQV`WtM+s
zm)BE>&^i^xCPGAry`31AaBduQboxH5(u0sW8m?5a-^I+l4@D=*1xwrqQz&;oOi<?H
z3B?j#=R8`&%ChzD<6g-@Cu|$<ejc8(p`rxiR18&0HZl^8^G$AoV$k5BMJKUgW4C4b
zsbH0asyL?Lx=&8;_Y5v<;EHuu%R2lSbQl+@!PvVTnrFKDCz(qZ)j%nY_igu{TGR-4
zsAj*wWuzsMb+R6uO<vJ^1ix*ZJJc1&QBRf>_ebs0Y|#y^c9EP@eHAr9Ow@aJyzrMd
z_YI396=eJytjekRMGJ4V8V7OOj-pD-()wfVV6D~c!kmJc4_gj4BRi76-ABIjoXvU{
z2Ar;c7)`g1ANO|Wpwm{Z8+mY^vtuUT+A<S-%mPG+t_Su?SP6c@>x27YTigYS&Rc|I
zGue)~#QjPVK8|{<;yc)^Sqv;L65?c0Lj5{2{IXoxA7p)rgTK$<%M4^Lf1a@)Veq32
z{sIC&!{xezhgjm?=6d3Qj`wl&6$Z~TSZ1(<z)$m{ul0V=3Wb{mpFD)P?7ghXZE+>p
z0#U>pq65a+l^zeftQ%Iy|0merk0Yp^&`;wraG$<r9sBXP_*KP1zJtrdT@9;Q$0Op4
zx6`c8c31UZW;b78@KX$a8bLe?lIPNcHIlPD>e=jXkfG@<(LclB=NbGQgAW+|H3na2
z@RtzyWi%X~1M0uXtb?1@NkWy00Uv73@-bYINCr3%$D;p>CE)MDz{K3I4+{P4O}h)Z
zo3<G<xF3GuCZ|}}-BrK3UB3G;m~fyu*zUi{@%}9amKJb%nzykNKSx6?;^+{`!5cMM
zQU4+bn@jRAE)c??M&jb&Ahu;$ei*T<Cw%%>*^{i<2aJ(f^)E0O0+9aeOv;MND$9Dy
z%72XoVrh&$sKI=RoJ0zC_EuqIRAF22%!U7W^SYQiS{V~xxb>4*Ur84B2mj$<5`L1+
zlFe8aK6CoW<)6oV+Aa(y<T;uiPmQIDDfn6VPH!A_@iuSL(|-?R@zZZyy#9)xp1S(_
zl%KhF{pz(TKXvZHH3n}m_?(|cE;27%yn*Ps>o+W;(UE00RbpK1;a#^JI;Mm?k6-vX
z1UQp}69i{+bpOJor{Iq1r`Jx%G3XIE^)e<6jWvt=nJGAUa1sbT8ZKj`voATf55wh_
zg-)8>#wMeS+YG*sTym)^&cZrg@A0~Gq-#09V4<Y0=KOM_*$jF-DaMd}87(f*!)Qf_
z8$~O;@ny56gIot^-YBv_SmwKAOGg_@dKl18Sb!lHvfP0~tBmc;yN|f~A0Yj$pZ+QS
z{r=f+vD9(?=<J7o1RxnYwg}X0DC_eGy1|DLaBsq4&Sxrlrt!=mjTL~K(ouP6=<eDd
z<Hk3>X}Ey+t#3l;m+1sE6$SSuY$0463VoZBi$7~pRveZ+hccvW+3n}(*qm_Wg3o)A
zhr5F13H0JRbf&RKgs{%U<ti2wj=}PHXSF{fHPF+8HB?oNyjwu6(f%H|+F*h7$6#5)
zio$zvS?Bh1;)okDd-mg9{k^ab_wosAEsed;FBT#ky!FRHDJ9xl{e5Pi!ic}<sZn~&
zO<^g$Bd56JyzDHN?{X%*-t#st6FIQ%;O3pfs=@-=)&oX@{?x=5L2Ig{_Iwe&_9x7Q
zqLbIQgDVj2(zKkCD?`7E^3h>Az8M^rkI6fK59MRS@-)k_Hhb}gaX6u6HQ~axajY9w
zWj}4pWc~KRS}mx3?_$4Y+<CUY{aixx`=<y^@)yg=un3}Ow#3hDi9dTOt}Hh?D?n2)
zC2nvCuf!F~dFoSgZz3+r<!yVtq9Ab!7?GD7fX*)bKlpHOeiB%XQ_O)ZOAe`TK<)Wu
zjK?a0<ZwB)NiK~KWzq>0)G{JO?8)Qy(}NHH5B_SSalS>TPj@vG2_0vk1$H7qC#zSE
zp_{mH0bf~M;#U^JVsZ8kzSIdq{r6djyZYTmr2h^FLNQENRujC9&ns|55+bK|z<xLK
zcUk3^SfykIkci(nf64mE!(GL1>B`kNFJ7;|0*t&6mtMbiZ8ITbt;;8IB81`;h1-a%
z>v^a&aw7)ICmDm6__VBWS$SAoW_nP{+dYGl*y@S!sdZQ5aw1M8B_}bQMDN)|XyPSs
z+HYzE0$rQ<HAei}E_BC<U~gbCoI)Z%fMFBvVf^!OR8~&$Ix~;T?31xP^1T!UP6`LM
zsWT8gM-u@CA?8gcqK5)#!qYK&9)ceA>LKX;5`^Km13iG<4{rfGvjyyO0%#Br;YlzM
zQUYi<Zl6p5EsxVi+=Il$=&gW;dZ<m%;u%r`XaIn+z!gEw-Vw|LrariR(!vw~XcQ1N
z%BLCK22pziqUbgnGh+!vRfZ6?J!cc5@&XrST+_C!UKJ>*Zto$1s4;=4yc!XR0z@jD
zUhRM=fTyZRp9<xKYuoPtqF8QkJ-&nTO$ZZ+0tDgSrK|~|h$~beB)$t#ya}MUPbNUL
zcN2*8uVX*;KW6Y73?5_fPZ7j<d<qxv{=mn93C<+C9emFm5s?Tbzrwa+PyQ3ehImrT
zek^v_lgEF7>hMVaOSTg2{<j#TUbBfD`d=|gv8?|!0|JKrHw=bwq5mzDzs*3fVtfZy
z=zqtef6rhS?Ogv33;qLx-(~Ps2LF-4?;*g)c>`$qPt5xU1Hr-HXKV+PPCUq@RP%i@
zDi_Gf4;Ynfx|K=6oWh#8GqEdc4m3k*hhWYBioS+d0jR4D^)-6J;4uRkDVY8sZfIfj
zOe@@`n5Tq>pLyJ|2m4@v6#pNCIp}M=qb0{#`ah$Go#gR1m?vAzrH4`cx0xXul46Xz
z0#UIakGM+1W{Zhbyg3B581cb+#_r9X%Xi3@@i-DBhA);wx!K4i{7Eih9^E@$H8qhM
z^YkAe?icaV+rWk`b@08&Faby&CaV0r0M_83aIe7m_RJkK_V;euzr*4Cqgyhi4}WA!
z4yC~GEWC)}(a&=pKe?{BZBsIEj&qBkx?t&X3qev)Xpn-kC^&^f18h_t#|sJ8VYA@W
zo=c++uD@ZkaF6MqqkH|E4E`B|Z?{E7yG45kOR;sIMrB4@b4EC$EgATqFQ|SDw@`l*
z@AP3N+O?qFhz9z<aU}np!A|zreSg2oy=7G=UxjSqKae<;Z-C3eEf4R8enFBq;O6Zj
zb_q|2B0Fz*tNtP5zsBGnA$Tz3lX0%jPt@vl>+PCBEkDDXtm}Td(~g!jhJx>n!e*<t
zehT>-9~G0p<E;JIRf{r{!KU}t#aSS}vW0eQR=)Jr@3C&0U$g2{jN)70<sQE7jf<CG
zow|Dc;`wva7yaD1OP4O+yg2Qbu3ep;zBoO7`3)p0*Dqc<cjNNasp-#NzBcXWF5WzU
z<@E~}{Ynz4U%P(!>UH^q${(3J_u9q!^hMZn=WblRKJAw!W6<xg33I3QRg6Tw4Azs3
z9bs^s!LtmWXCUbNGGnhWxX9ow22%{03_j0bj=^;XL|WZv@D&CE+Je&58U&w&LuDJc
z(1B2d7-adEpu4TKgU{s9AS5vJ_9FlB?x{Spyqasj%d+2Q;T917jb-!MVy;-sr}CqS
zjaAF}%uegNd?jBQJxJ8#zw%@GTydY{77O{42p=8f51!>{+#RbrF3QeJ_<X*C>%oaF
G{(k{*BC&A*

diff --git a/Projects/NLP/SMS_Annotation/Input_Data/.DS_Store b/Projects/NLP/SMS_Annotation/Input_Data/.DS_Store
deleted file mode 100644
index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 6148
zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3
zem<@ulZcFPQ@L2!n>{z**<q8>++&mCkOWA81W14cNZ<zv;LbK1Poaz?KmsK2CSc!(
z0ynLxE!0092;Krf2c+FF_Fe*7ECH>lEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ
zLs35+`xjp>T0<F0fCPF1$Cyrb|F7^5{eNG?83~ZUUlGt@xh*qZDeu<Z%US-OSsOPv
j)R!Z4KLME7ReXlK;d!wEw5GODWMKRea10D2@KpjYNUI8I

diff --git a/README_local.md b/README_local.md
index 6e55771..430b09a 100644
--- a/README_local.md
+++ b/README_local.md
@@ -13,6 +13,8 @@ This document describes how to use 5 versions of name extraction scripts for vot
 
 - Alternatively, if you are in an environment where you can't / don't want to install Anaconda, create and activate your Python 3 virtualenv (see [pipenv and virtualenv](https://docs.python-guide.org/dev/virtualenvs/)) and run `pip install -r requirements.txt`
 
+- You'll also need to run `spacy download en` once.
+
 ## Getting Started
 Find your use case below and add your input data to the appropriate place, then run the specified python script.
 All of these scripts should be run out of the directory `Projects/NLP/SMS_Annotation`  
@@ -84,10 +86,10 @@ This script will output two files:
 Add a csv to the Input_Data folder. This csv file must contain column 'names' containing the names logged by a text banker  
 
 **Instructions:**
-In this directory, run `python3 Code/name_cleaning.py -d [input_filename]`  
+In this directory, run `python3 Code/name_cleaning.py -i [input_filename]`
 
 **Outputs:**
-A File named `labeled_names_cleaned_no_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file 
+A file in `Output_Data` named `labeled_names_cleaned_no_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file 
   
   
 ## Text Banker Log Cleaning (utilizing text message conversation)
@@ -128,3 +130,15 @@ This script will output two files:
 - *ContactName* a unique identifier for the conversation
 - *NoteText* free text possibly including names of tripling targets
 - *names_extract* a guess for the extracted names (to be reviewed)
+
+# Running the app frontend
+app.py is a Flask-based frontend that provides a dedicated UI for uploading data sets and running the scripts with them.
+
+Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
+
+To run an instance of the frontend locally:
+`
+export FLASK_APP=app.py
+export FLASK_ENV=development
+flask run
+`
\ No newline at end of file
diff --git a/app.py b/app.py
new file mode 100644
index 0000000..133ddb6
--- /dev/null
+++ b/app.py
@@ -0,0 +1,11 @@
+from flask import Flask, request, render_template
+app = Flask(__name__)
+
+@app.route('/')
+def index():
+    if request.method == 'POST':
+        import pdb; pdb.set_trace();
+        # handle file upload
+        return
+    else: 
+        return render_template('upload_form.html')
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index d84b090..fc5c90f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,5 @@ pathlib==1.0.1
 python-Levenshtein==0.12.0
 scikit-learn==0.23.2
 scipy==1.5.4
-spacy==2.3.4
\ No newline at end of file
+spacy==2.3.4
+Flask==1.1.2
\ No newline at end of file
diff --git a/templates/upload_form.html b/templates/upload_form.html
new file mode 100644
index 0000000..363344f
--- /dev/null
+++ b/templates/upload_form.html
@@ -0,0 +1,82 @@
+<html>
+
+<head>
+    <title>
+        Vote tripling SMS transcript parsing
+    </title>
+
+    <style>
+        body { width: 96%; padding: 2%; }
+        h1 h2 h3 h4 h5 h6 { font-family: 'sans-serif'; }
+        main { max-width: 800px; width: 100%; margin: auto; }
+        ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
+        ul.menu li a { padding: .5em; color: darkblue; background-color: #ccc;}
+    </style>
+</head>
+
+<body>
+
+    <main>
+
+        <h1>Vote tripling SMS transcript parsing</h1>
+
+        <ul class="menu"><li><a href="#van_export_cleaning">VAN export cleaning</a></li>
+            <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+        </ul>
+        
+
+        <hr />
+
+        <h3 id='van_export_cleaning'>VAN Export Cleaning</h3>
+         <p><strong>I have a VAN Export and I need to extract any tripling target names from the note text.</strong></p>
+
+         <p>Upload a csv file with the following columns:</p>
+         <ul>
+        <li>*VANID* a unique ID for this row
+        <li>*ContactName* the name of the tripler
+        <li>*NoteText* free text possibly including names of tripling targets
+        </ul>
+
+        <form method="post" enctype="multipart/form-data">
+            <input type="file" id="vec_file" name="vec_file" accept=".csv">
+            <button>Submit</button>
+        </form>
+
+        <p>If successful, the output will include two files: 
+        <ul><li>1. A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+            <ul><li>- *VANID* a unique identifier for the conversation
+                <li>- *names_extract* the extracted names
+            </ul>
+
+            <li>2. A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+                <ul><li>- *VANID* a unique identifier for the conversation
+                    <li>- *ContactName* a unique identifier for the conversation
+                    <li>- *NoteText* free text possibly including names of tripling targets
+                    <li>- *names_extract* a guess for the extracted names (to be reviewed)
+                </ul>
+        </ul>
+
+        <hr />
+
+        <h3 id="text_banker_log_cleaning"> Text Banker Log Cleaning (utilizing text message conversation)</h3>
+        <p><strong> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </strong></p>
+
+        <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
+
+        <p>Upload a csv file of the same format as the output of the aggregation in step 1.
+        This csv file must also contain column 'names' containing the names logged by a text banker.</p>
+
+        <form method="post" enctype="multipart/form-data">
+            <input type="file" id="tblc_file" name="tblc_file" accept=".csv">
+            <button>Submit</button>
+        </form>
+
+        <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+
+        <hr/>
+
+    </main>
+
+</body>
+
+</html>
\ No newline at end of file

From 3cdb61c556d2493393b8aaed84022dfdb2497b2c Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Sat, 2 Jan 2021 20:33:46 -0700
Subject: [PATCH 05/52] some thoughts on the backend

---
 .gitignore                 |   1 +
 app.py                     |  67 +++++++++++++++++--
 settings.py.example        |   3 +
 templates/upload_form.html | 127 +++++++++++++++++++++++++++++--------
 4 files changed, 166 insertions(+), 32 deletions(-)
 create mode 100644 settings.py.example

diff --git a/.gitignore b/.gitignore
index 525f974..e40a865 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 *.csv
 __pycache__
 *.pyc
+settings.py
diff --git a/app.py b/app.py
index 133ddb6..6cb8fce 100644
--- a/app.py
+++ b/app.py
@@ -1,11 +1,66 @@
-from flask import Flask, request, render_template
+from flask import (flash, Flask, g, redirect, render_template, request,
+    send_from_directory, url_for)
+from werkzeug.utils import secure_filename
+import os
+import settings
+import sqlite3
+
+# configure app
 app = Flask(__name__)
+APP_ROOT = os.path.dirname(os.path.abspath(__file__))
+app.config['UPLOAD_FOLDER'] = os.path.join(APP_ROOT, 'Projects/NLP/SMS_Annotation/Input_data')
+
+ALLOWED_EXTENSIONS = settings.ALLOWED_EXTENSIONS
+app.secret_key = settings.SECRET_KEY
+app.config['MAX_CONTENT_LENGTH'] = settings.MAX_CONTENT_LENGTH
+
+
+# check filename extension on uploaded files
+def allowed_file(filename):
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
+def queue_job(type, message):
+    return
+
+def process_job(job_id):
+    return
+
+def complete_job(job_id):
+    return
 
-@app.route('/')
+def delete_file(filename):
+    return
+
+# most of the business is here
+@app.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
-        import pdb; pdb.set_trace();
-        # handle file upload
-        return
+        # check if the post request has the file part
+        if 'vec_file' not in request.files:
+            flash('No file part')
+            return redirect(request.url)
+        file = request.files['vec_file']
+        # if user does not select file, browser also
+        # submit an empty part without filename
+        if file.filename == '':
+            flash('No selected file')
+            return redirect(request.url)
+        if file and allowed_file(file.filename):
+            filename = secure_filename(file.filename)
+            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
+            return redirect(url_for('uploaded_file',
+                                    filename=filename))
     else: 
-        return render_template('upload_form.html')
\ No newline at end of file
+        return render_template('upload_form.html')
+
+# returning uploaded files
+@app.route('/uploads/<filename>')
+def uploaded_file(filename):
+    return send_from_directory(app.config['UPLOAD_FOLDER'],
+                               filename)
+    # TODO: documentation on setting file directory permissions
+    # i.e. uploads dir can't be inspected
+
+
+
+
diff --git a/settings.py.example b/settings.py.example
new file mode 100644
index 0000000..9a77ab3
--- /dev/null
+++ b/settings.py.example
@@ -0,0 +1,3 @@
+ALLOWED_EXTENSIONS = {'csv'}
+SECRET_KEY = '' # keep this secret in production!
+MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16 MB
diff --git a/templates/upload_form.html b/templates/upload_form.html
index 363344f..15db708 100644
--- a/templates/upload_form.html
+++ b/templates/upload_form.html
@@ -5,12 +5,19 @@
         Vote tripling SMS transcript parsing
     </title>
 
+    <link href="https://fonts.googleapis.com/css?family=Droid+Serif|Roboto:500" rel="stylesheet">
     <style>
-        body { width: 96%; padding: 2%; }
-        h1 h2 h3 h4 h5 h6 { font-family: 'sans-serif'; }
-        main { max-width: 800px; width: 100%; margin: auto; }
+        body { width: 100%; font-family: 'Droid Serif', serif; font-size: 16px; background-color: lightblue; }
+        h1,h2,h3,h4,h5,h6 { font-family: 'Roboto', sans-serif !important; }
+        main { max-width: 700px; margin: auto; background-color: #eff; padding: 2em; }
+        a { text-decoration: none; color: royalblue; }
+        a:hover { text-decoration: underline; color: purple; }
+        a:visited { color: rebeccapurple; }
         ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
-        ul.menu li a { padding: .5em; color: darkblue; background-color: #ccc;}
+        ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
+        ul.menu li a:hover { color: purple; border: 2px solid purple; }
+        input, button { font-size: .9em; height: 2em; }
+        blockquote { display: block;  margin-left: 0; padding: 1em; background-color: rgba(255,255,255,.8); }
     </style>
 </head>
 
@@ -20,46 +27,112 @@
 
         <h1>Vote tripling SMS transcript parsing</h1>
 
-        <ul class="menu"><li><a href="#van_export_cleaning">VAN export cleaning</a></li>
+        <ul class="menu">
+            <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
             <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+            <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
         </ul>
         
 
+<!--        ## SMS Aggregation
+**Use Case:** I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.
+
+**Inputs:**
+Add a csv added to the Input_Data folder. This csv should be raw individual SMS messages, not grouped by conversation.
+
+**Instructions:**
+Open the script aggregate_text_messages.R in RStudioo and follow the instructions to aggregate messages into a single row per conversation
+
+**Outputs:**
+A file (filename specified by you in the R script) with a single row representing each text message conversation, including the following fields
+- *ConversationId* a unique identifier for the conversation
+- *contact_phone* the phone number of the target
+- *totalMessages* the total number of messages exchanged
+- *tripleMessage* initial message sent from the text banker to the target
+- *voterResponse* initial response(s) by the target (generally where the target makes known if they opt out or want to triple)
+- *tripleResponse* follow up message sent from the text banker to the target
+- *voterFinal* the final follow up message sent by the target (generally where they provide names)
+- *tripleFinal* final follow up sent by text banker
+- *voterPost* post script from the target (generally a thank you or good luck)
+- *noResponse* boolean for whether there was no response
+- *negResponse* boolean for generally negative or discouraging terms (sorry, no, etc.)
+- *posResponse* boolean for generally positive or encouraging terms
+- *affirmResponse* boolean for presence of a scripted affirmation by text banker
+- *finalAffirmResponse* boolean for presence of a scripted follow up affirmation by text banker
+-->
         <hr />
+        <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
+        <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
 
-        <h3 id='van_export_cleaning'>VAN Export Cleaning</h3>
-         <p><strong>I have a VAN Export and I need to extract any tripling target names from the note text.</strong></p>
+        <p>Upload a csv file of the same format as the output of the aggregation in step 1.</p>
+        
+        <form method="post" enctype="multipart/form-data">
+            <input type=file id="sccne_file" name="sccne_file" accept=".csv">
+            <input type=submit value=Upload>
+        </form>
+
+        <p>If successful, the output will include two files:
+        <ul>
+            <li>A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+            <ul>
+                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                <li><strong>contact_phone:</strong> the phone number of the target</li>
+                <li><strong>is_tripler:</strong> did this person agree to be a tripler ('yes' for everyone in this file)</li>
+                <li><strong>opted_out:</strong> did this person opt out of future messages</li>
+                <li><strong>wrong_number:</strong> did we have the wrong number for this person</li>
+                <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
+            </ul>
+
+            <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+            <ul>
+                <li><strong>ConversationId:</strong> a unique identifier for the conversation
+                <li><strong>contact_phone:</strong> the phone number of the target</li>
+                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                <li><strong>is_tripler:</strong> guess for did this person agree to be a tripler (to be reviewed)</li>
+                <li><strong>opted_out:</strong> guess for did this person opt out of future messages (to be reviewed)</li>
+                <li><strong>wrong_number:</strong> guess for did we have the wrong number for this person (to be reviewed)</li>
+                <li><strong>names_extract:</strong> guess for what names (if any) were provided by this person as tripling targets (to be reviewed)</li>
+            </ul>
+        </ul>
+
+        <hr />
+
+        <h2 id='van_export_cleaning'>VAN Export Cleaning</h2>
+        <blockquote>I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
 
-         <p>Upload a csv file with the following columns:</p>
-         <ul>
-        <li>*VANID* a unique ID for this row
-        <li>*ContactName* the name of the tripler
-        <li>*NoteText* free text possibly including names of tripling targets
+        <p>Upload a csv file with the following columns:</p>
+        <ul>
+            <li><strong>VANID:</strong> a unique ID for this row
+            <li><strong>ContactName:</strong> the name of the tripler
+            <li><strong>NoteText:</strong> free text possibly including names of tripling targets
         </ul>
 
         <form method="post" enctype="multipart/form-data">
-            <input type="file" id="vec_file" name="vec_file" accept=".csv">
-            <button>Submit</button>
+            <input type=file id="vec_file" name="vec_file" accept=".csv">
+            <input type=submit value=Upload>
         </form>
 
         <p>If successful, the output will include two files: 
-        <ul><li>1. A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
-            <ul><li>- *VANID* a unique identifier for the conversation
-                <li>- *names_extract* the extracted names
+        <ul><li>A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+            <ul><li><strong>VANID:</strong> a unique identifier for the conversation
+                <li><strong>names_extract:</strong> the extracted names
             </ul>
 
-            <li>2. A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
-                <ul><li>- *VANID* a unique identifier for the conversation
-                    <li>- *ContactName* a unique identifier for the conversation
-                    <li>- *NoteText* free text possibly including names of tripling targets
-                    <li>- *names_extract* a guess for the extracted names (to be reviewed)
+            <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+                <ul><li><strong>VANID:</strong> a unique identifier for the conversation
+                    <li><strong>ContactName:</strong> a unique identifier for the conversation
+                    <li><strong>NoteText:</strong> free text possibly including names of tripling targets
+                    <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)
                 </ul>
         </ul>
 
         <hr />
 
-        <h3 id="text_banker_log_cleaning"> Text Banker Log Cleaning (utilizing text message conversation)</h3>
-        <p><strong> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </strong></p>
+        <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text message conversation)</h2>
+
+        <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
 
         <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
 
@@ -67,14 +140,16 @@ <h3 id="text_banker_log_cleaning"> Text Banker Log Cleaning (utilizing text mess
         This csv file must also contain column 'names' containing the names logged by a text banker.</p>
 
         <form method="post" enctype="multipart/form-data">
-            <input type="file" id="tblc_file" name="tblc_file" accept=".csv">
-            <button>Submit</button>
+            <input type=file id="tblc_file" name="tblc_file" accept=".csv">
+            <input type=submit value=Upload>
         </form>
 
         <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
 
         <hr/>
 
+
+
     </main>
 
 </body>

From 95c8fde5f1b5ce9bbcac552ca726401523add5a4 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Sat, 2 Jan 2021 20:33:46 -0700
Subject: [PATCH 06/52] some thoughts on the backend

---
 .gitignore                 |   1 +
 app.py                     |  67 +++++++++++++++++--
 settings.py.example        |   3 +
 templates/upload_form.html | 127 +++++++++++++++++++++++++++++--------
 4 files changed, 166 insertions(+), 32 deletions(-)
 create mode 100644 settings.py.example

diff --git a/.gitignore b/.gitignore
index 525f974..e40a865 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 *.csv
 __pycache__
 *.pyc
+settings.py
diff --git a/app.py b/app.py
index 133ddb6..6cb8fce 100644
--- a/app.py
+++ b/app.py
@@ -1,11 +1,66 @@
-from flask import Flask, request, render_template
+from flask import (flash, Flask, g, redirect, render_template, request,
+    send_from_directory, url_for)
+from werkzeug.utils import secure_filename
+import os
+import settings
+import sqlite3
+
+# configure app
 app = Flask(__name__)
+APP_ROOT = os.path.dirname(os.path.abspath(__file__))
+app.config['UPLOAD_FOLDER'] = os.path.join(APP_ROOT, 'Projects/NLP/SMS_Annotation/Input_data')
+
+ALLOWED_EXTENSIONS = settings.ALLOWED_EXTENSIONS
+app.secret_key = settings.SECRET_KEY
+app.config['MAX_CONTENT_LENGTH'] = settings.MAX_CONTENT_LENGTH
+
+
+# check filename extension on uploaded files
+def allowed_file(filename):
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
+def queue_job(type, message):
+    return
+
+def process_job(job_id):
+    return
+
+def complete_job(job_id):
+    return
 
-@app.route('/')
+def delete_file(filename):
+    return
+
+# most of the business is here
+@app.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
-        import pdb; pdb.set_trace();
-        # handle file upload
-        return
+        # check if the post request has the file part
+        if 'vec_file' not in request.files:
+            flash('No file part')
+            return redirect(request.url)
+        file = request.files['vec_file']
+        # if user does not select file, browser also
+        # submit an empty part without filename
+        if file.filename == '':
+            flash('No selected file')
+            return redirect(request.url)
+        if file and allowed_file(file.filename):
+            filename = secure_filename(file.filename)
+            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
+            return redirect(url_for('uploaded_file',
+                                    filename=filename))
     else: 
-        return render_template('upload_form.html')
\ No newline at end of file
+        return render_template('upload_form.html')
+
+# returning uploaded files
+@app.route('/uploads/<filename>')
+def uploaded_file(filename):
+    return send_from_directory(app.config['UPLOAD_FOLDER'],
+                               filename)
+    # TODO: documentation on setting file directory permissions
+    # i.e. uploads dir can't be inspected
+
+
+
+
diff --git a/settings.py.example b/settings.py.example
new file mode 100644
index 0000000..9a77ab3
--- /dev/null
+++ b/settings.py.example
@@ -0,0 +1,3 @@
+ALLOWED_EXTENSIONS = {'csv'}
+SECRET_KEY = '' # keep this secret in production!
+MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16 MB
diff --git a/templates/upload_form.html b/templates/upload_form.html
index 363344f..15db708 100644
--- a/templates/upload_form.html
+++ b/templates/upload_form.html
@@ -5,12 +5,19 @@
         Vote tripling SMS transcript parsing
     </title>
 
+    <link href="https://fonts.googleapis.com/css?family=Droid+Serif|Roboto:500" rel="stylesheet">
     <style>
-        body { width: 96%; padding: 2%; }
-        h1 h2 h3 h4 h5 h6 { font-family: 'sans-serif'; }
-        main { max-width: 800px; width: 100%; margin: auto; }
+        body { width: 100%; font-family: 'Droid Serif', serif; font-size: 16px; background-color: lightblue; }
+        h1,h2,h3,h4,h5,h6 { font-family: 'Roboto', sans-serif !important; }
+        main { max-width: 700px; margin: auto; background-color: #eff; padding: 2em; }
+        a { text-decoration: none; color: royalblue; }
+        a:hover { text-decoration: underline; color: purple; }
+        a:visited { color: rebeccapurple; }
         ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
-        ul.menu li a { padding: .5em; color: darkblue; background-color: #ccc;}
+        ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
+        ul.menu li a:hover { color: purple; border: 2px solid purple; }
+        input, button { font-size: .9em; height: 2em; }
+        blockquote { display: block;  margin-left: 0; padding: 1em; background-color: rgba(255,255,255,.8); }
     </style>
 </head>
 
@@ -20,46 +27,112 @@
 
         <h1>Vote tripling SMS transcript parsing</h1>
 
-        <ul class="menu"><li><a href="#van_export_cleaning">VAN export cleaning</a></li>
+        <ul class="menu">
+            <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
             <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+            <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
         </ul>
         
 
+<!--        ## SMS Aggregation
+**Use Case:** I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.
+
+**Inputs:**
+Add a csv added to the Input_Data folder. This csv should be raw individual SMS messages, not grouped by conversation.
+
+**Instructions:**
+Open the script aggregate_text_messages.R in RStudioo and follow the instructions to aggregate messages into a single row per conversation
+
+**Outputs:**
+A file (filename specified by you in the R script) with a single row representing each text message conversation, including the following fields
+- *ConversationId* a unique identifier for the conversation
+- *contact_phone* the phone number of the target
+- *totalMessages* the total number of messages exchanged
+- *tripleMessage* initial message sent from the text banker to the target
+- *voterResponse* initial response(s) by the target (generally where the target makes known if they opt out or want to triple)
+- *tripleResponse* follow up message sent from the text banker to the target
+- *voterFinal* the final follow up message sent by the target (generally where they provide names)
+- *tripleFinal* final follow up sent by text banker
+- *voterPost* post script from the target (generally a thank you or good luck)
+- *noResponse* boolean for whether there was no response
+- *negResponse* boolean for generally negative or discouraging terms (sorry, no, etc.)
+- *posResponse* boolean for generally positive or encouraging terms
+- *affirmResponse* boolean for presence of a scripted affirmation by text banker
+- *finalAffirmResponse* boolean for presence of a scripted follow up affirmation by text banker
+-->
         <hr />
+        <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
+        <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
 
-        <h3 id='van_export_cleaning'>VAN Export Cleaning</h3>
-         <p><strong>I have a VAN Export and I need to extract any tripling target names from the note text.</strong></p>
+        <p>Upload a csv file of the same format as the output of the aggregation in step 1.</p>
+        
+        <form method="post" enctype="multipart/form-data">
+            <input type=file id="sccne_file" name="sccne_file" accept=".csv">
+            <input type=submit value=Upload>
+        </form>
+
+        <p>If successful, the output will include two files:
+        <ul>
+            <li>A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+            <ul>
+                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                <li><strong>contact_phone:</strong> the phone number of the target</li>
+                <li><strong>is_tripler:</strong> did this person agree to be a tripler ('yes' for everyone in this file)</li>
+                <li><strong>opted_out:</strong> did this person opt out of future messages</li>
+                <li><strong>wrong_number:</strong> did we have the wrong number for this person</li>
+                <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
+            </ul>
+
+            <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+            <ul>
+                <li><strong>ConversationId:</strong> a unique identifier for the conversation
+                <li><strong>contact_phone:</strong> the phone number of the target</li>
+                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                <li><strong>is_tripler:</strong> guess for did this person agree to be a tripler (to be reviewed)</li>
+                <li><strong>opted_out:</strong> guess for did this person opt out of future messages (to be reviewed)</li>
+                <li><strong>wrong_number:</strong> guess for did we have the wrong number for this person (to be reviewed)</li>
+                <li><strong>names_extract:</strong> guess for what names (if any) were provided by this person as tripling targets (to be reviewed)</li>
+            </ul>
+        </ul>
+
+        <hr />
+
+        <h2 id='van_export_cleaning'>VAN Export Cleaning</h2>
+        <blockquote>I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
 
-         <p>Upload a csv file with the following columns:</p>
-         <ul>
-        <li>*VANID* a unique ID for this row
-        <li>*ContactName* the name of the tripler
-        <li>*NoteText* free text possibly including names of tripling targets
+        <p>Upload a csv file with the following columns:</p>
+        <ul>
+            <li><strong>VANID:</strong> a unique ID for this row
+            <li><strong>ContactName:</strong> the name of the tripler
+            <li><strong>NoteText:</strong> free text possibly including names of tripling targets
         </ul>
 
         <form method="post" enctype="multipart/form-data">
-            <input type="file" id="vec_file" name="vec_file" accept=".csv">
-            <button>Submit</button>
+            <input type=file id="vec_file" name="vec_file" accept=".csv">
+            <input type=submit value=Upload>
         </form>
 
         <p>If successful, the output will include two files: 
-        <ul><li>1. A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
-            <ul><li>- *VANID* a unique identifier for the conversation
-                <li>- *names_extract* the extracted names
+        <ul><li>A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+            <ul><li><strong>VANID:</strong> a unique identifier for the conversation
+                <li><strong>names_extract:</strong> the extracted names
             </ul>
 
-            <li>2. A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
-                <ul><li>- *VANID* a unique identifier for the conversation
-                    <li>- *ContactName* a unique identifier for the conversation
-                    <li>- *NoteText* free text possibly including names of tripling targets
-                    <li>- *names_extract* a guess for the extracted names (to be reviewed)
+            <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+                <ul><li><strong>VANID:</strong> a unique identifier for the conversation
+                    <li><strong>ContactName:</strong> a unique identifier for the conversation
+                    <li><strong>NoteText:</strong> free text possibly including names of tripling targets
+                    <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)
                 </ul>
         </ul>
 
         <hr />
 
-        <h3 id="text_banker_log_cleaning"> Text Banker Log Cleaning (utilizing text message conversation)</h3>
-        <p><strong> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </strong></p>
+        <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text message conversation)</h2>
+
+        <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
 
         <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
 
@@ -67,14 +140,16 @@ <h3 id="text_banker_log_cleaning"> Text Banker Log Cleaning (utilizing text mess
         This csv file must also contain column 'names' containing the names logged by a text banker.</p>
 
         <form method="post" enctype="multipart/form-data">
-            <input type="file" id="tblc_file" name="tblc_file" accept=".csv">
-            <button>Submit</button>
+            <input type=file id="tblc_file" name="tblc_file" accept=".csv">
+            <input type=submit value=Upload>
         </form>
 
         <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
 
         <hr/>
 
+
+
     </main>
 
 </body>

From 2c8ce6de741452e5d8eeebfe6aaf6e26688dd978 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Mon, 4 Jan 2021 09:09:51 -0700
Subject: [PATCH 07/52] updated UI with temporarily disabled forms

---
 templates/upload_form.html | 220 ++++++++++++++++++++-----------------
 1 file changed, 119 insertions(+), 101 deletions(-)

diff --git a/templates/upload_form.html b/templates/upload_form.html
index 15db708..163325b 100644
--- a/templates/upload_form.html
+++ b/templates/upload_form.html
@@ -18,6 +18,8 @@
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
         input, button { font-size: .9em; height: 2em; }
         blockquote { display: block;  margin-left: 0; padding: 1em; background-color: rgba(255,255,255,.8); }
+        div.upload-disabled { opacity: .3; }
+        div.upload-disabled input {display: none;}
     </style>
 </head>
 
@@ -25,128 +27,144 @@
 
     <main>
 
-        <h1>Vote tripling SMS transcript parsing</h1>
+        <div>
+            <h1>Vote tripling SMS transcript parsing</h1>
 
-        <ul class="menu">
-            <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
-            <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
-            <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
-        </ul>
-        
+            <ul class="menu">
+                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
+                <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+                <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
+            </ul>
+        </div>
 
-<!--        ## SMS Aggregation
-**Use Case:** I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.
-
-**Inputs:**
-Add a csv added to the Input_Data folder. This csv should be raw individual SMS messages, not grouped by conversation.
-
-**Instructions:**
-Open the script aggregate_text_messages.R in RStudioo and follow the instructions to aggregate messages into a single row per conversation
-
-**Outputs:**
-A file (filename specified by you in the R script) with a single row representing each text message conversation, including the following fields
-- *ConversationId* a unique identifier for the conversation
-- *contact_phone* the phone number of the target
-- *totalMessages* the total number of messages exchanged
-- *tripleMessage* initial message sent from the text banker to the target
-- *voterResponse* initial response(s) by the target (generally where the target makes known if they opt out or want to triple)
-- *tripleResponse* follow up message sent from the text banker to the target
-- *voterFinal* the final follow up message sent by the target (generally where they provide names)
-- *tripleFinal* final follow up sent by text banker
-- *voterPost* post script from the target (generally a thank you or good luck)
-- *noResponse* boolean for whether there was no response
-- *negResponse* boolean for generally negative or discouraging terms (sorry, no, etc.)
-- *posResponse* boolean for generally positive or encouraging terms
-- *affirmResponse* boolean for presence of a scripted affirmation by text banker
-- *finalAffirmResponse* boolean for presence of a scripted follow up affirmation by text banker
--->
-        <hr />
-        <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
-        <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
-
-        <p>Upload a csv file of the same format as the output of the aggregation in step 1.</p>
-        
-        <form method="post" enctype="multipart/form-data">
-            <input type=file id="sccne_file" name="sccne_file" accept=".csv">
-            <input type=submit value=Upload>
-        </form>
-
-        <p>If successful, the output will include two files:
-        <ul>
-            <li>A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+        <div class="upload-enabled">
+            <hr />
+
+            <h2 id='van_export_cleaning'>VAN Export Cleaning</h2>
+            <blockquote>I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
+
+            <p>Upload a csv file with the following columns:</p>
             <ul>
-                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
-                <li><strong>contact_phone:</strong> the phone number of the target</li>
-                <li><strong>is_tripler:</strong> did this person agree to be a tripler ('yes' for everyone in this file)</li>
-                <li><strong>opted_out:</strong> did this person opt out of future messages</li>
-                <li><strong>wrong_number:</strong> did we have the wrong number for this person</li>
-                <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
+                <li><strong>VANID:</strong> a unique ID for this row
+                <li><strong>ContactName:</strong> the name of the tripler
+                <li><strong>NoteText:</strong> free text possibly including names of tripling targets
             </ul>
 
-            <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="vec_file" name="vec_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
+
+            <p>If successful, the output will include two files: 
             <ul>
-                <li><strong>ConversationId:</strong> a unique identifier for the conversation
-                <li><strong>contact_phone:</strong> the phone number of the target</li>
-                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
-                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
-                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
-                <li><strong>is_tripler:</strong> guess for did this person agree to be a tripler (to be reviewed)</li>
-                <li><strong>opted_out:</strong> guess for did this person opt out of future messages (to be reviewed)</li>
-                <li><strong>wrong_number:</strong> guess for did we have the wrong number for this person (to be reviewed)</li>
-                <li><strong>names_extract:</strong> guess for what names (if any) were provided by this person as tripling targets (to be reviewed)</li>
+                <li>A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+                <ul>
+                    <li><strong>VANID:</strong> a unique identifier for the conversation</li>
+                    <li><strong>names_extract:</strong> the extracted names</li>
+                </ul>
+
+                <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:</li>
+                <ul>
+                    <li><strong>VANID:</strong> a unique identifier for the conversation</li>
+                    <li><strong>ContactName:</strong> a unique identifier for the conversation</li>
+                    <li><strong>NoteText:</strong> free text possibly including names of tripling targets</li>
+                    <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)</li>
+                </ul>
             </ul>
-        </ul>
+        </  div>
 
-        <hr />
+        <div class="upload-disabled">
+            <hr />
 
-        <h2 id='van_export_cleaning'>VAN Export Cleaning</h2>
-        <blockquote>I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
+            <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text message conversation)</h2>
 
-        <p>Upload a csv file with the following columns:</p>
-        <ul>
-            <li><strong>VANID:</strong> a unique ID for this row
-            <li><strong>ContactName:</strong> the name of the tripler
-            <li><strong>NoteText:</strong> free text possibly including names of tripling targets
-        </ul>
+            <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
 
-        <form method="post" enctype="multipart/form-data">
-            <input type=file id="vec_file" name="vec_file" accept=".csv">
-            <input type=submit value=Upload>
-        </form>
+            <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
 
-        <p>If successful, the output will include two files: 
-        <ul><li>A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
-            <ul><li><strong>VANID:</strong> a unique identifier for the conversation
-                <li><strong>names_extract:</strong> the extracted names
-            </ul>
+            <p>Upload a csv file of the same format as the output of the aggregation in step 1.
+            This csv file must also contain column 'names' containing the names logged by a text banker.</p>
 
-            <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
-                <ul><li><strong>VANID:</strong> a unique identifier for the conversation
-                    <li><strong>ContactName:</strong> a unique identifier for the conversation
-                    <li><strong>NoteText:</strong> free text possibly including names of tripling targets
-                    <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)
-                </ul>
-        </ul>
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="tblc_file" name="tblc_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
 
-        <hr />
+            <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+
+        </div>
+        
+        <div class="upload-disabled">
+            <hr />
+            <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
+            <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
+
+            <p>Upload a csv file of the same format as the output of the aggregation in step 1.</p>
+            
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="sccne_file" name="sccne_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
+
+            <p>If successful, the output will include two files:
+            <ul>
+                <li>A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+                <ul>
+                    <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                    <li><strong>contact_phone:</strong> the phone number of the target</li>
+                    <li><strong>is_tripler:</strong> did this person agree to be a tripler ('yes' for everyone in this file)</li>
+                    <li><strong>opted_out:</strong> did this person opt out of future messages</li>
+                    <li><strong>wrong_number:</strong> did we have the wrong number for this person</li>
+                    <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
+                </ul>
 
-        <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text message conversation)</h2>
+                <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+                <ul>
+                    <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                    <li><strong>contact_phone:</strong> the phone number of the target</li>
+                    <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                    <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                    <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                    <li><strong>is_tripler:</strong> guess for did this person agree to be a tripler (to be reviewed)</li>
+                    <li><strong>opted_out:</strong> guess for did this person opt out of future messages (to be reviewed)</li>
+                    <li><strong>wrong_number:</strong> guess for did we have the wrong number for this person (to be reviewed)</li>
+                    <li><strong>names_extract:</strong> guess for what names (if any) were provided by this person as tripling targets (to be reviewed)</li>
+                </ul>
+            </ul>
+        </div>
 
-        <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
+        <div class="upload-disabled">
+            <hr />
 
-        <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
+            <h2 id="sms_agg">SMS Aggregation</h2>
+            <blockquote>I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.</blockquote>
 
-        <p>Upload a csv file of the same format as the output of the aggregation in step 1.
-        This csv file must also contain column 'names' containing the names logged by a text banker.</p>
+            <p>Upload a csv file of raw individual SMS messages, not grouped by conversation.</p>
 
-        <form method="post" enctype="multipart/form-data">
-            <input type=file id="tblc_file" name="tblc_file" accept=".csv">
-            <input type=submit value=Upload>
-        </form>
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="sms_agg_file" name="sms_agg_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
 
-        <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+            <p>If successful, the output will be a csv file with a single row representing each text message conversation, including the following fields:
+            <ul>
+                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                <li><strong>contact_phone:</strong> the phone number of the target</li>
+                <li><strong>totalMessages:</strong> the total number of messages exchanged</li>
+                <li><strong>tripleMessage:</strong> initial message sent from the text banker to the target</li>
+                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                <li><strong>tripleResponse:</strong> follow up message sent from the text banker to the target</li>
+                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                <li><strong>tripleFinal:</strong> final follow up sent by text banker</li>
+                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                <li><strong>noResponse:</strong> boolean for whether there was no response</li>
+                <li><strong>negResponse:</strong> boolean for generally negative or discouraging terms (sorry, no, etc.)</li>
+                <li><strong>posResponse:</strong> boolean for generally positive or encouraging terms</li>
+                <li><strong>affirmResponse:</strong> boolean for presence of a scripted affirmation by text banker</li>
+                <li><strong>finalAffirmResponse:</strong> boolean for presence of a scripted follow up affirmation by text banker</li>
+            </ul>
 
-        <hr/>
+        </div>
 
 
 

From 7b8a95d6fff8172378376bffe58f6caab263dd91 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Mon, 4 Jan 2021 09:09:51 -0700
Subject: [PATCH 08/52] updated UI with temporarily disabled forms

---
 templates/upload_form.html | 220 ++++++++++++++++++++-----------------
 1 file changed, 119 insertions(+), 101 deletions(-)

diff --git a/templates/upload_form.html b/templates/upload_form.html
index 15db708..163325b 100644
--- a/templates/upload_form.html
+++ b/templates/upload_form.html
@@ -18,6 +18,8 @@
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
         input, button { font-size: .9em; height: 2em; }
         blockquote { display: block;  margin-left: 0; padding: 1em; background-color: rgba(255,255,255,.8); }
+        div.upload-disabled { opacity: .3; }
+        div.upload-disabled input {display: none;}
     </style>
 </head>
 
@@ -25,128 +27,144 @@
 
     <main>
 
-        <h1>Vote tripling SMS transcript parsing</h1>
+        <div>
+            <h1>Vote tripling SMS transcript parsing</h1>
 
-        <ul class="menu">
-            <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
-            <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
-            <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
-        </ul>
-        
+            <ul class="menu">
+                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
+                <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+                <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
+            </ul>
+        </div>
 
-<!--        ## SMS Aggregation
-**Use Case:** I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.
-
-**Inputs:**
-Add a csv added to the Input_Data folder. This csv should be raw individual SMS messages, not grouped by conversation.
-
-**Instructions:**
-Open the script aggregate_text_messages.R in RStudioo and follow the instructions to aggregate messages into a single row per conversation
-
-**Outputs:**
-A file (filename specified by you in the R script) with a single row representing each text message conversation, including the following fields
-- *ConversationId* a unique identifier for the conversation
-- *contact_phone* the phone number of the target
-- *totalMessages* the total number of messages exchanged
-- *tripleMessage* initial message sent from the text banker to the target
-- *voterResponse* initial response(s) by the target (generally where the target makes known if they opt out or want to triple)
-- *tripleResponse* follow up message sent from the text banker to the target
-- *voterFinal* the final follow up message sent by the target (generally where they provide names)
-- *tripleFinal* final follow up sent by text banker
-- *voterPost* post script from the target (generally a thank you or good luck)
-- *noResponse* boolean for whether there was no response
-- *negResponse* boolean for generally negative or discouraging terms (sorry, no, etc.)
-- *posResponse* boolean for generally positive or encouraging terms
-- *affirmResponse* boolean for presence of a scripted affirmation by text banker
-- *finalAffirmResponse* boolean for presence of a scripted follow up affirmation by text banker
--->
-        <hr />
-        <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
-        <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
-
-        <p>Upload a csv file of the same format as the output of the aggregation in step 1.</p>
-        
-        <form method="post" enctype="multipart/form-data">
-            <input type=file id="sccne_file" name="sccne_file" accept=".csv">
-            <input type=submit value=Upload>
-        </form>
-
-        <p>If successful, the output will include two files:
-        <ul>
-            <li>A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+        <div class="upload-enabled">
+            <hr />
+
+            <h2 id='van_export_cleaning'>VAN Export Cleaning</h2>
+            <blockquote>I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
+
+            <p>Upload a csv file with the following columns:</p>
             <ul>
-                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
-                <li><strong>contact_phone:</strong> the phone number of the target</li>
-                <li><strong>is_tripler:</strong> did this person agree to be a tripler ('yes' for everyone in this file)</li>
-                <li><strong>opted_out:</strong> did this person opt out of future messages</li>
-                <li><strong>wrong_number:</strong> did we have the wrong number for this person</li>
-                <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
+                <li><strong>VANID:</strong> a unique ID for this row
+                <li><strong>ContactName:</strong> the name of the tripler
+                <li><strong>NoteText:</strong> free text possibly including names of tripling targets
             </ul>
 
-            <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="vec_file" name="vec_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
+
+            <p>If successful, the output will include two files: 
             <ul>
-                <li><strong>ConversationId:</strong> a unique identifier for the conversation
-                <li><strong>contact_phone:</strong> the phone number of the target</li>
-                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
-                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
-                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
-                <li><strong>is_tripler:</strong> guess for did this person agree to be a tripler (to be reviewed)</li>
-                <li><strong>opted_out:</strong> guess for did this person opt out of future messages (to be reviewed)</li>
-                <li><strong>wrong_number:</strong> guess for did we have the wrong number for this person (to be reviewed)</li>
-                <li><strong>names_extract:</strong> guess for what names (if any) were provided by this person as tripling targets (to be reviewed)</li>
+                <li>A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+                <ul>
+                    <li><strong>VANID:</strong> a unique identifier for the conversation</li>
+                    <li><strong>names_extract:</strong> the extracted names</li>
+                </ul>
+
+                <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:</li>
+                <ul>
+                    <li><strong>VANID:</strong> a unique identifier for the conversation</li>
+                    <li><strong>ContactName:</strong> a unique identifier for the conversation</li>
+                    <li><strong>NoteText:</strong> free text possibly including names of tripling targets</li>
+                    <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)</li>
+                </ul>
             </ul>
-        </ul>
+        </  div>
 
-        <hr />
+        <div class="upload-disabled">
+            <hr />
 
-        <h2 id='van_export_cleaning'>VAN Export Cleaning</h2>
-        <blockquote>I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
+            <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text message conversation)</h2>
 
-        <p>Upload a csv file with the following columns:</p>
-        <ul>
-            <li><strong>VANID:</strong> a unique ID for this row
-            <li><strong>ContactName:</strong> the name of the tripler
-            <li><strong>NoteText:</strong> free text possibly including names of tripling targets
-        </ul>
+            <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
 
-        <form method="post" enctype="multipart/form-data">
-            <input type=file id="vec_file" name="vec_file" accept=".csv">
-            <input type=submit value=Upload>
-        </form>
+            <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
 
-        <p>If successful, the output will include two files: 
-        <ul><li>A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
-            <ul><li><strong>VANID:</strong> a unique identifier for the conversation
-                <li><strong>names_extract:</strong> the extracted names
-            </ul>
+            <p>Upload a csv file of the same format as the output of the aggregation in step 1.
+            This csv file must also contain column 'names' containing the names logged by a text banker.</p>
 
-            <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
-                <ul><li><strong>VANID:</strong> a unique identifier for the conversation
-                    <li><strong>ContactName:</strong> a unique identifier for the conversation
-                    <li><strong>NoteText:</strong> free text possibly including names of tripling targets
-                    <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)
-                </ul>
-        </ul>
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="tblc_file" name="tblc_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
 
-        <hr />
+            <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+
+        </div>
+        
+        <div class="upload-disabled">
+            <hr />
+            <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
+            <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
+
+            <p>Upload a csv file of the same format as the output of the aggregation in step 1.</p>
+            
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="sccne_file" name="sccne_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
+
+            <p>If successful, the output will include two files:
+            <ul>
+                <li>A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+                <ul>
+                    <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                    <li><strong>contact_phone:</strong> the phone number of the target</li>
+                    <li><strong>is_tripler:</strong> did this person agree to be a tripler ('yes' for everyone in this file)</li>
+                    <li><strong>opted_out:</strong> did this person opt out of future messages</li>
+                    <li><strong>wrong_number:</strong> did we have the wrong number for this person</li>
+                    <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
+                </ul>
 
-        <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text message conversation)</h2>
+                <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+                <ul>
+                    <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                    <li><strong>contact_phone:</strong> the phone number of the target</li>
+                    <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                    <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                    <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                    <li><strong>is_tripler:</strong> guess for did this person agree to be a tripler (to be reviewed)</li>
+                    <li><strong>opted_out:</strong> guess for did this person opt out of future messages (to be reviewed)</li>
+                    <li><strong>wrong_number:</strong> guess for did we have the wrong number for this person (to be reviewed)</li>
+                    <li><strong>names_extract:</strong> guess for what names (if any) were provided by this person as tripling targets (to be reviewed)</li>
+                </ul>
+            </ul>
+        </div>
 
-        <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
+        <div class="upload-disabled">
+            <hr />
 
-        <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
+            <h2 id="sms_agg">SMS Aggregation</h2>
+            <blockquote>I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.</blockquote>
 
-        <p>Upload a csv file of the same format as the output of the aggregation in step 1.
-        This csv file must also contain column 'names' containing the names logged by a text banker.</p>
+            <p>Upload a csv file of raw individual SMS messages, not grouped by conversation.</p>
 
-        <form method="post" enctype="multipart/form-data">
-            <input type=file id="tblc_file" name="tblc_file" accept=".csv">
-            <input type=submit value=Upload>
-        </form>
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="sms_agg_file" name="sms_agg_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
 
-        <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+            <p>If successful, the output will be a csv file with a single row representing each text message conversation, including the following fields:
+            <ul>
+                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                <li><strong>contact_phone:</strong> the phone number of the target</li>
+                <li><strong>totalMessages:</strong> the total number of messages exchanged</li>
+                <li><strong>tripleMessage:</strong> initial message sent from the text banker to the target</li>
+                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                <li><strong>tripleResponse:</strong> follow up message sent from the text banker to the target</li>
+                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                <li><strong>tripleFinal:</strong> final follow up sent by text banker</li>
+                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                <li><strong>noResponse:</strong> boolean for whether there was no response</li>
+                <li><strong>negResponse:</strong> boolean for generally negative or discouraging terms (sorry, no, etc.)</li>
+                <li><strong>posResponse:</strong> boolean for generally positive or encouraging terms</li>
+                <li><strong>affirmResponse:</strong> boolean for presence of a scripted affirmation by text banker</li>
+                <li><strong>finalAffirmResponse:</strong> boolean for presence of a scripted follow up affirmation by text banker</li>
+            </ul>
 
-        <hr/>
+        </div>
 
 
 

From e170abf5f0a6ba5a08e01af554325b04f1ae2569 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 7 Jan 2021 10:36:20 -0700
Subject: [PATCH 09/52] commenting

---
 app.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/app.py b/app.py
index 6cb8fce..2552744 100644
--- a/app.py
+++ b/app.py
@@ -19,6 +19,8 @@
 def allowed_file(filename):
     return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
+# For now, we're going to go ahead and run the script as soon as we get a file upload
+# Ultimately we want to use a queue to make sure we're doing these one at a time.
 def queue_job(type, message):
     return
 
@@ -35,13 +37,14 @@ def delete_file(filename):
 @app.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
-        # check if the post request has the file part
+        import pdb; pdb.set_trace()
+        # TODO: check to see if a file was submitted at all
         if 'vec_file' not in request.files:
             flash('No file part')
             return redirect(request.url)
+        # TODO: if multiple files are submitted, take the first one
         file = request.files['vec_file']
-        # if user does not select file, browser also
-        # submit an empty part without filename
+        # If selected file is empty / without filename
         if file.filename == '':
             flash('No selected file')
             return redirect(request.url)
@@ -50,6 +53,9 @@ def index():
             file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
             return redirect(url_for('uploaded_file',
                                     filename=filename))
+        # TODO: check file for required headers
+        # TODO: run script
+        # TODO: return output file
     else: 
         return render_template('upload_form.html')
 

From 535e3df9207dcb5146e6f00a6ea4d61d9b4a0f00 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 7 Jan 2021 10:36:20 -0700
Subject: [PATCH 10/52] commenting

---
 app.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/app.py b/app.py
index 6cb8fce..2552744 100644
--- a/app.py
+++ b/app.py
@@ -19,6 +19,8 @@
 def allowed_file(filename):
     return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
+# For now, we're going to go ahead and run the script as soon as we get a file upload
+# Ultimately we want to use a queue to make sure we're doing these one at a time.
 def queue_job(type, message):
     return
 
@@ -35,13 +37,14 @@ def delete_file(filename):
 @app.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
-        # check if the post request has the file part
+        import pdb; pdb.set_trace()
+        # TODO: check to see if a file was submitted at all
         if 'vec_file' not in request.files:
             flash('No file part')
             return redirect(request.url)
+        # TODO: if multiple files are submitted, take the first one
         file = request.files['vec_file']
-        # if user does not select file, browser also
-        # submit an empty part without filename
+        # If selected file is empty / without filename
         if file.filename == '':
             flash('No selected file')
             return redirect(request.url)
@@ -50,6 +53,9 @@ def index():
             file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
             return redirect(url_for('uploaded_file',
                                     filename=filename))
+        # TODO: check file for required headers
+        # TODO: run script
+        # TODO: return output file
     else: 
         return render_template('upload_form.html')
 

From c876292b7de7b3ddaeb8b598d65ca2efa0b39cc4 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 26 Feb 2021 12:32:19 -0700
Subject: [PATCH 11/52] added tests and checks

---
 app.py                     | 55 +++++++++++++++++++++++++-------------
 requirements.txt           |  4 ++-
 templates/upload_form.html | 11 ++++++++
 tests/test_flaskr.py       | 19 +++++++++++++
 4 files changed, 69 insertions(+), 20 deletions(-)
 create mode 100644 tests/test_flaskr.py

diff --git a/app.py b/app.py
index 2552744..7a0a7df 100644
--- a/app.py
+++ b/app.py
@@ -1,6 +1,8 @@
 from flask import (flash, Flask, g, redirect, render_template, request,
     send_from_directory, url_for)
 from werkzeug.utils import secure_filename
+
+import csv
 import os
 import settings
 import sqlite3
@@ -13,11 +15,21 @@
 ALLOWED_EXTENSIONS = settings.ALLOWED_EXTENSIONS
 app.secret_key = settings.SECRET_KEY
 app.config['MAX_CONTENT_LENGTH'] = settings.MAX_CONTENT_LENGTH
-
+REQUIRED_HEADERS = settings.REQUIRED_HEADERS
 
 # check filename extension on uploaded files
-def allowed_file(filename):
-    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+def allowed_file(file):
+    return ('.' in file.filename 
+        and file.filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+        and file.content_type == 'text/csv')
+
+def check_headers(file, upload_type = 'vec_file'):
+    # check that the required headers are present in the file
+    headers = file.readline().decode('utf-8').split(',')
+    for header in REQUIRED_HEADERS[upload_type]:
+        if header not in headers:
+            return False
+    return True
 
 # For now, we're going to go ahead and run the script as soon as we get a file upload
 # Ultimately we want to use a queue to make sure we're doing these one at a time.
@@ -37,27 +49,32 @@ def delete_file(filename):
 @app.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
-        import pdb; pdb.set_trace()
-        # TODO: check to see if a file was submitted at all
-        if 'vec_file' not in request.files:
-            flash('No file part')
+        # make sure there's a REQUIRED_HEADERS entry for each file upload input
+        if not (REQUIRED_HEADERS.keys() & request.files.keys()):
+            flash('No file', 'error')
             return redirect(request.url)
-        # TODO: if multiple files are submitted, take the first one
-        file = request.files['vec_file']
-        # If selected file is empty / without filename
-        if file.filename == '':
-            flash('No selected file')
-            return redirect(request.url)
-        if file and allowed_file(file.filename):
+        # iterate through the different file uploads
+        for upload_type in request.files:
+            file = request.files[upload_type]
+            # If selected file is empty / without filename
+            if file.filename == '':
+                flash('No selected file', 'error')
+                return redirect(request.url)
+            if not (file and allowed_file(file)):
+                flash('Invalid file type. Select a CSV to upload', 'error')
+                return render_template('upload_form.html')
+            if not check_headers(file, 'vec_file'):
+                flash('Invalid CSV headers. Make sure uploaded CSV has required columns.', 'error')
+                return render_template('upload_form.html')
+            # if it passes all the checks, save file and queue for processing
             filename = secure_filename(file.filename)
             file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
+            # TODO: queue for processing instead of spitting it back
             return redirect(url_for('uploaded_file',
                                     filename=filename))
-        # TODO: check file for required headers
-        # TODO: run script
-        # TODO: return output file
-    else: 
-        return render_template('upload_form.html')
+
+    # GET
+    return render_template('upload_form.html')
 
 # returning uploaded files
 @app.route('/uploads/<filename>')
diff --git a/requirements.txt b/requirements.txt
index fc5c90f..496c04e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,12 @@
+coverage==5.4
+Flask==1.1.2
 matplotlib==3.3.3
 nltk==3.5
 numpy==1.19.4
 pandas==1.1.5
 pathlib==1.0.1
+pytest==6.2.2
 python-Levenshtein==0.12.0
 scikit-learn==0.23.2
 scipy==1.5.4
 spacy==2.3.4
-Flask==1.1.2
\ No newline at end of file
diff --git a/templates/upload_form.html b/templates/upload_form.html
index 163325b..42f12bc 100644
--- a/templates/upload_form.html
+++ b/templates/upload_form.html
@@ -13,6 +13,7 @@
         a { text-decoration: none; color: royalblue; }
         a:hover { text-decoration: underline; color: purple; }
         a:visited { color: rebeccapurple; }
+        ul.flashes li {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
         ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
         ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
@@ -27,6 +28,16 @@
 
     <main>
 
+    {% with messages = get_flashed_messages() %}
+      {% if messages %}
+        <ul class=flashes>
+        {% for message in messages %}
+          <li>{{ message }}</li>
+        {% endfor %}
+        </ul>
+      {% endif %}
+    {% endwith %}
+
         <div>
             <h1>Vote tripling SMS transcript parsing</h1>
 
diff --git a/tests/test_flaskr.py b/tests/test_flaskr.py
new file mode 100644
index 0000000..755ab69
--- /dev/null
+++ b/tests/test_flaskr.py
@@ -0,0 +1,19 @@
+import os
+import tempfile
+
+import pytest
+
+from flaskr import flaskr
+
+@pytest.fixture
+def client():
+    db_fd, flaskr.app.config['DATABASE'] = tempfile.mkstemp()
+    flaskr.app.config['TESTING'] = True
+
+    with flaskr.app.test_client() as client:
+        with flaskr.app.app_context():
+            flaskr.init_db()
+        yield client
+
+    os.close(db_fd)
+    os.unlink(flaskr.app.config['DATABASE'])
\ No newline at end of file

From dc2523d318f3f68cfe7d1d747884d0bfd499ef8b Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 26 Feb 2021 12:32:19 -0700
Subject: [PATCH 12/52] added tests and checks

---
 app.py                     | 55 +++++++++++++++++++++++++-------------
 requirements.txt           |  4 ++-
 templates/upload_form.html | 11 ++++++++
 tests/test_flaskr.py       | 19 +++++++++++++
 4 files changed, 69 insertions(+), 20 deletions(-)
 create mode 100644 tests/test_flaskr.py

diff --git a/app.py b/app.py
index 2552744..7a0a7df 100644
--- a/app.py
+++ b/app.py
@@ -1,6 +1,8 @@
 from flask import (flash, Flask, g, redirect, render_template, request,
     send_from_directory, url_for)
 from werkzeug.utils import secure_filename
+
+import csv
 import os
 import settings
 import sqlite3
@@ -13,11 +15,21 @@
 ALLOWED_EXTENSIONS = settings.ALLOWED_EXTENSIONS
 app.secret_key = settings.SECRET_KEY
 app.config['MAX_CONTENT_LENGTH'] = settings.MAX_CONTENT_LENGTH
-
+REQUIRED_HEADERS = settings.REQUIRED_HEADERS
 
 # check filename extension on uploaded files
-def allowed_file(filename):
-    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+def allowed_file(file):
+    return ('.' in file.filename 
+        and file.filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+        and file.content_type == 'text/csv')
+
+def check_headers(file, upload_type = 'vec_file'):
+    # check that the required headers are present in the file
+    headers = file.readline().decode('utf-8').split(',')
+    for header in REQUIRED_HEADERS[upload_type]:
+        if header not in headers:
+            return False
+    return True
 
 # For now, we're going to go ahead and run the script as soon as we get a file upload
 # Ultimately we want to use a queue to make sure we're doing these one at a time.
@@ -37,27 +49,32 @@ def delete_file(filename):
 @app.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
-        import pdb; pdb.set_trace()
-        # TODO: check to see if a file was submitted at all
-        if 'vec_file' not in request.files:
-            flash('No file part')
+        # make sure there's a REQUIRED_HEADERS entry for each file upload input
+        if not (REQUIRED_HEADERS.keys() & request.files.keys()):
+            flash('No file', 'error')
             return redirect(request.url)
-        # TODO: if multiple files are submitted, take the first one
-        file = request.files['vec_file']
-        # If selected file is empty / without filename
-        if file.filename == '':
-            flash('No selected file')
-            return redirect(request.url)
-        if file and allowed_file(file.filename):
+        # iterate through the different file uploads
+        for upload_type in request.files:
+            file = request.files[upload_type]
+            # If selected file is empty / without filename
+            if file.filename == '':
+                flash('No selected file', 'error')
+                return redirect(request.url)
+            if not (file and allowed_file(file)):
+                flash('Invalid file type. Select a CSV to upload', 'error')
+                return render_template('upload_form.html')
+            if not check_headers(file, 'vec_file'):
+                flash('Invalid CSV headers. Make sure uploaded CSV has required columns.', 'error')
+                return render_template('upload_form.html')
+            # if it passes all the checks, save file and queue for processing
             filename = secure_filename(file.filename)
             file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
+            # TODO: queue for processing instead of spitting it back
             return redirect(url_for('uploaded_file',
                                     filename=filename))
-        # TODO: check file for required headers
-        # TODO: run script
-        # TODO: return output file
-    else: 
-        return render_template('upload_form.html')
+
+    # GET
+    return render_template('upload_form.html')
 
 # returning uploaded files
 @app.route('/uploads/<filename>')
diff --git a/requirements.txt b/requirements.txt
index fc5c90f..496c04e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,12 @@
+coverage==5.4
+Flask==1.1.2
 matplotlib==3.3.3
 nltk==3.5
 numpy==1.19.4
 pandas==1.1.5
 pathlib==1.0.1
+pytest==6.2.2
 python-Levenshtein==0.12.0
 scikit-learn==0.23.2
 scipy==1.5.4
 spacy==2.3.4
-Flask==1.1.2
\ No newline at end of file
diff --git a/templates/upload_form.html b/templates/upload_form.html
index 163325b..42f12bc 100644
--- a/templates/upload_form.html
+++ b/templates/upload_form.html
@@ -13,6 +13,7 @@
         a { text-decoration: none; color: royalblue; }
         a:hover { text-decoration: underline; color: purple; }
         a:visited { color: rebeccapurple; }
+        ul.flashes li {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
         ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
         ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
@@ -27,6 +28,16 @@
 
     <main>
 
+    {% with messages = get_flashed_messages() %}
+      {% if messages %}
+        <ul class=flashes>
+        {% for message in messages %}
+          <li>{{ message }}</li>
+        {% endfor %}
+        </ul>
+      {% endif %}
+    {% endwith %}
+
         <div>
             <h1>Vote tripling SMS transcript parsing</h1>
 
diff --git a/tests/test_flaskr.py b/tests/test_flaskr.py
new file mode 100644
index 0000000..755ab69
--- /dev/null
+++ b/tests/test_flaskr.py
@@ -0,0 +1,19 @@
+import os
+import tempfile
+
+import pytest
+
+from flaskr import flaskr
+
+@pytest.fixture
+def client():
+    db_fd, flaskr.app.config['DATABASE'] = tempfile.mkstemp()
+    flaskr.app.config['TESTING'] = True
+
+    with flaskr.app.test_client() as client:
+        with flaskr.app.app_context():
+            flaskr.init_db()
+        yield client
+
+    os.close(db_fd)
+    os.unlink(flaskr.app.config['DATABASE'])
\ No newline at end of file

From aff5d1e68645168514a1a63a28f07fd031a07e95 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Mon, 1 Mar 2021 11:34:01 -0700
Subject: [PATCH 13/52] refactored as a factory

---
 .gitignore                                    | 13 +++++-
 parser/__init__.py                            | 37 ++++++++++++++++
 parser/db.py                                  | 43 +++++++++++++++++++
 app.py => parser/main.py                      | 29 ++++++-------
 parser/schema.sql                             | 10 +++++
 .../templates}/upload_form.html               |  0
 parser/tests/test_parse.py                    | 19 ++++++++
 tests/test_flaskr.py                          | 19 --------
 8 files changed, 135 insertions(+), 35 deletions(-)
 create mode 100644 parser/__init__.py
 create mode 100644 parser/db.py
 rename app.py => parser/main.py (82%)
 create mode 100644 parser/schema.sql
 rename {templates => parser/templates}/upload_form.html (100%)
 create mode 100644 parser/tests/test_parse.py
 delete mode 100644 tests/test_flaskr.py

diff --git a/.gitignore b/.gitignore
index e40a865..36dba15 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,16 @@
 .DS_Store
 *.csv
+settings.py
+
+instance/
+
 __pycache__
 *.pyc
-settings.py
+
+.pytest_cache/
+.coverage
+htmlcov/
+
+dist/
+build/
+*.egg-info/
diff --git a/parser/__init__.py b/parser/__init__.py
new file mode 100644
index 0000000..24daca3
--- /dev/null
+++ b/parser/__init__.py
@@ -0,0 +1,37 @@
+import os
+import settings
+
+from flask import Flask
+
+
+def create_app(test_config=None):
+    # create and configure the app
+    app = Flask(__name__, instance_relative_config=True)
+    APP_ROOT = os.path.dirname(os.path.abspath(__file__))
+    app.config.from_mapping(
+        SECRET_KEY=settings.SECRET_KEY,
+        DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
+        MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
+        UPLOAD_FOLDER=os.path.join(APP_ROOT, settings.UPLOAD_FOLDER)
+    )
+
+    from . import db
+    db.init_app(app)
+
+    from . import main
+    app.register_blueprint(main.bp)
+
+    if test_config is None:
+        # load the instance config, if it exists, when not testing
+        app.config.from_pyfile('config.py', silent=True)
+    else:
+        # load the test config if passed in
+        app.config.from_mapping(test_config)
+
+    # ensure the instance folder exists
+    try:
+        os.makedirs(app.instance_path)
+    except OSError:
+        pass
+
+    return app
\ No newline at end of file
diff --git a/parser/db.py b/parser/db.py
new file mode 100644
index 0000000..0b4f188
--- /dev/null
+++ b/parser/db.py
@@ -0,0 +1,43 @@
+import sqlite3
+
+import click
+from flask import current_app, g
+from flask.cli import with_appcontext
+
+
+def get_db():
+    if 'db' not in g:
+        g.db = sqlite3.connect(
+            current_app.config['DATABASE'],
+            detect_types=sqlite3.PARSE_DECLTYPES
+        )
+        g.db.row_factory = sqlite3.Row
+
+    return g.db
+
+
+def close_db(e=None):
+    db = g.pop('db', None)
+
+    if db is not None:
+        db.close()
+
+
+def init_db():
+    db = get_db()
+
+    with current_app.open_resource('schema.sql') as f:
+        db.executescript(f.read().decode('utf8'))
+
+
+@click.command('init-db')
+@with_appcontext
+def init_db_command():
+    """Clear the existing data and create new tables."""
+    init_db()
+    click.echo('Initialized the database.')
+
+
+def init_app(app):
+    app.teardown_appcontext(close_db)
+    app.cli.add_command(init_db_command)
\ No newline at end of file
diff --git a/app.py b/parser/main.py
similarity index 82%
rename from app.py
rename to parser/main.py
index 7a0a7df..e7b46ae 100644
--- a/app.py
+++ b/parser/main.py
@@ -1,21 +1,20 @@
-from flask import (flash, Flask, g, redirect, render_template, request,
-    send_from_directory, url_for)
-from werkzeug.utils import secure_filename
-
+import functools
 import csv
 import os
 import settings
 import sqlite3
 
-# configure app
-app = Flask(__name__)
-APP_ROOT = os.path.dirname(os.path.abspath(__file__))
-app.config['UPLOAD_FOLDER'] = os.path.join(APP_ROOT, 'Projects/NLP/SMS_Annotation/Input_data')
+from flask import (Blueprint, flash, Flask, g, redirect, render_template, request,
+    send_from_directory, url_for)
+from parser.db import get_db
+from werkzeug.utils import secure_filename
+
+bp = Blueprint('main', __name__, url_prefix='')
 
-ALLOWED_EXTENSIONS = settings.ALLOWED_EXTENSIONS
-app.secret_key = settings.SECRET_KEY
-app.config['MAX_CONTENT_LENGTH'] = settings.MAX_CONTENT_LENGTH
-REQUIRED_HEADERS = settings.REQUIRED_HEADERS
+ALLOWED_EXTENSIONS = {'csv'}
+REQUIRED_HEADERS = {
+    'vec_file': ['names']
+}
 
 # check filename extension on uploaded files
 def allowed_file(file):
@@ -46,7 +45,7 @@ def delete_file(filename):
     return
 
 # most of the business is here
-@app.route('/', methods=['GET', 'POST'])
+@bp.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
         # make sure there's a REQUIRED_HEADERS entry for each file upload input
@@ -76,8 +75,8 @@ def index():
     # GET
     return render_template('upload_form.html')
 
-# returning uploaded files
-@app.route('/uploads/<filename>')
+# returning uploaded files; get rid of this in prod
+@bp.route('/uploads/<filename>')
 def uploaded_file(filename):
     return send_from_directory(app.config['UPLOAD_FOLDER'],
                                filename)
diff --git a/parser/schema.sql b/parser/schema.sql
new file mode 100644
index 0000000..dd99695
--- /dev/null
+++ b/parser/schema.sql
@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS jobs;
+
+CREATE TABLE jobs (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  file_path TEXT NOT NULL,
+  job_type TEXT NOT NULL,
+  email TEXT NOT NULL,
+  status TEXT NOT NULL
+);
diff --git a/templates/upload_form.html b/parser/templates/upload_form.html
similarity index 100%
rename from templates/upload_form.html
rename to parser/templates/upload_form.html
diff --git a/parser/tests/test_parse.py b/parser/tests/test_parse.py
new file mode 100644
index 0000000..19020e3
--- /dev/null
+++ b/parser/tests/test_parse.py
@@ -0,0 +1,19 @@
+import os
+import tempfile
+
+import pytest
+
+from . import parser
+
+@pytest.fixture
+def client():
+    db_fd, parser.app.config['DATABASE'] = tempfile.mkstemp()
+    parser.app.config['TESTING'] = True
+
+    with parser.app.test_client() as client:
+        with parser.app.app_context():
+            parser.init_db()
+        yield client
+
+    os.close(db_fd)
+    os.unlink(parser.app.config['DATABASE'])
\ No newline at end of file
diff --git a/tests/test_flaskr.py b/tests/test_flaskr.py
deleted file mode 100644
index 755ab69..0000000
--- a/tests/test_flaskr.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import os
-import tempfile
-
-import pytest
-
-from flaskr import flaskr
-
-@pytest.fixture
-def client():
-    db_fd, flaskr.app.config['DATABASE'] = tempfile.mkstemp()
-    flaskr.app.config['TESTING'] = True
-
-    with flaskr.app.test_client() as client:
-        with flaskr.app.app_context():
-            flaskr.init_db()
-        yield client
-
-    os.close(db_fd)
-    os.unlink(flaskr.app.config['DATABASE'])
\ No newline at end of file

From e06da64e63908d3063e456677c15d432099c1b67 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Mon, 1 Mar 2021 11:34:01 -0700
Subject: [PATCH 14/52] refactored as a factory

---
 .gitignore                                    | 13 +++++-
 parser/__init__.py                            | 37 ++++++++++++++++
 parser/db.py                                  | 43 +++++++++++++++++++
 app.py => parser/main.py                      | 29 ++++++-------
 parser/schema.sql                             | 10 +++++
 .../templates}/upload_form.html               |  0
 parser/tests/test_parse.py                    | 19 ++++++++
 tests/test_flaskr.py                          | 19 --------
 8 files changed, 135 insertions(+), 35 deletions(-)
 create mode 100644 parser/__init__.py
 create mode 100644 parser/db.py
 rename app.py => parser/main.py (82%)
 create mode 100644 parser/schema.sql
 rename {templates => parser/templates}/upload_form.html (100%)
 create mode 100644 parser/tests/test_parse.py
 delete mode 100644 tests/test_flaskr.py

diff --git a/.gitignore b/.gitignore
index e40a865..36dba15 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,16 @@
 .DS_Store
 *.csv
+settings.py
+
+instance/
+
 __pycache__
 *.pyc
-settings.py
+
+.pytest_cache/
+.coverage
+htmlcov/
+
+dist/
+build/
+*.egg-info/
diff --git a/parser/__init__.py b/parser/__init__.py
new file mode 100644
index 0000000..24daca3
--- /dev/null
+++ b/parser/__init__.py
@@ -0,0 +1,37 @@
+import os
+import settings
+
+from flask import Flask
+
+
+def create_app(test_config=None):
+    # create and configure the app
+    app = Flask(__name__, instance_relative_config=True)
+    APP_ROOT = os.path.dirname(os.path.abspath(__file__))
+    app.config.from_mapping(
+        SECRET_KEY=settings.SECRET_KEY,
+        DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
+        MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
+        UPLOAD_FOLDER=os.path.join(APP_ROOT, settings.UPLOAD_FOLDER)
+    )
+
+    from . import db
+    db.init_app(app)
+
+    from . import main
+    app.register_blueprint(main.bp)
+
+    if test_config is None:
+        # load the instance config, if it exists, when not testing
+        app.config.from_pyfile('config.py', silent=True)
+    else:
+        # load the test config if passed in
+        app.config.from_mapping(test_config)
+
+    # ensure the instance folder exists
+    try:
+        os.makedirs(app.instance_path)
+    except OSError:
+        pass
+
+    return app
\ No newline at end of file
diff --git a/parser/db.py b/parser/db.py
new file mode 100644
index 0000000..0b4f188
--- /dev/null
+++ b/parser/db.py
@@ -0,0 +1,43 @@
+import sqlite3
+
+import click
+from flask import current_app, g
+from flask.cli import with_appcontext
+
+
+def get_db():
+    if 'db' not in g:
+        g.db = sqlite3.connect(
+            current_app.config['DATABASE'],
+            detect_types=sqlite3.PARSE_DECLTYPES
+        )
+        g.db.row_factory = sqlite3.Row
+
+    return g.db
+
+
+def close_db(e=None):
+    db = g.pop('db', None)
+
+    if db is not None:
+        db.close()
+
+
+def init_db():
+    db = get_db()
+
+    with current_app.open_resource('schema.sql') as f:
+        db.executescript(f.read().decode('utf8'))
+
+
+@click.command('init-db')
+@with_appcontext
+def init_db_command():
+    """Clear the existing data and create new tables."""
+    init_db()
+    click.echo('Initialized the database.')
+
+
+def init_app(app):
+    app.teardown_appcontext(close_db)
+    app.cli.add_command(init_db_command)
\ No newline at end of file
diff --git a/app.py b/parser/main.py
similarity index 82%
rename from app.py
rename to parser/main.py
index 7a0a7df..e7b46ae 100644
--- a/app.py
+++ b/parser/main.py
@@ -1,21 +1,20 @@
-from flask import (flash, Flask, g, redirect, render_template, request,
-    send_from_directory, url_for)
-from werkzeug.utils import secure_filename
-
+import functools
 import csv
 import os
 import settings
 import sqlite3
 
-# configure app
-app = Flask(__name__)
-APP_ROOT = os.path.dirname(os.path.abspath(__file__))
-app.config['UPLOAD_FOLDER'] = os.path.join(APP_ROOT, 'Projects/NLP/SMS_Annotation/Input_data')
+from flask import (Blueprint, flash, Flask, g, redirect, render_template, request,
+    send_from_directory, url_for)
+from parser.db import get_db
+from werkzeug.utils import secure_filename
+
+bp = Blueprint('main', __name__, url_prefix='')
 
-ALLOWED_EXTENSIONS = settings.ALLOWED_EXTENSIONS
-app.secret_key = settings.SECRET_KEY
-app.config['MAX_CONTENT_LENGTH'] = settings.MAX_CONTENT_LENGTH
-REQUIRED_HEADERS = settings.REQUIRED_HEADERS
+ALLOWED_EXTENSIONS = {'csv'}
+REQUIRED_HEADERS = {
+    'vec_file': ['names']
+}
 
 # check filename extension on uploaded files
 def allowed_file(file):
@@ -46,7 +45,7 @@ def delete_file(filename):
     return
 
 # most of the business is here
-@app.route('/', methods=['GET', 'POST'])
+@bp.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
         # make sure there's a REQUIRED_HEADERS entry for each file upload input
@@ -76,8 +75,8 @@ def index():
     # GET
     return render_template('upload_form.html')
 
-# returning uploaded files
-@app.route('/uploads/<filename>')
+# returning uploaded files; get rid of this in prod
+@bp.route('/uploads/<filename>')
 def uploaded_file(filename):
     return send_from_directory(app.config['UPLOAD_FOLDER'],
                                filename)
diff --git a/parser/schema.sql b/parser/schema.sql
new file mode 100644
index 0000000..dd99695
--- /dev/null
+++ b/parser/schema.sql
@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS jobs;
+
+CREATE TABLE jobs (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  file_path TEXT NOT NULL,
+  job_type TEXT NOT NULL,
+  email TEXT NOT NULL,
+  status TEXT NOT NULL
+);
diff --git a/templates/upload_form.html b/parser/templates/upload_form.html
similarity index 100%
rename from templates/upload_form.html
rename to parser/templates/upload_form.html
diff --git a/parser/tests/test_parse.py b/parser/tests/test_parse.py
new file mode 100644
index 0000000..19020e3
--- /dev/null
+++ b/parser/tests/test_parse.py
@@ -0,0 +1,19 @@
+import os
+import tempfile
+
+import pytest
+
+from . import parser
+
+@pytest.fixture
+def client():
+    db_fd, parser.app.config['DATABASE'] = tempfile.mkstemp()
+    parser.app.config['TESTING'] = True
+
+    with parser.app.test_client() as client:
+        with parser.app.app_context():
+            parser.init_db()
+        yield client
+
+    os.close(db_fd)
+    os.unlink(parser.app.config['DATABASE'])
\ No newline at end of file
diff --git a/tests/test_flaskr.py b/tests/test_flaskr.py
deleted file mode 100644
index 755ab69..0000000
--- a/tests/test_flaskr.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import os
-import tempfile
-
-import pytest
-
-from flaskr import flaskr
-
-@pytest.fixture
-def client():
-    db_fd, flaskr.app.config['DATABASE'] = tempfile.mkstemp()
-    flaskr.app.config['TESTING'] = True
-
-    with flaskr.app.test_client() as client:
-        with flaskr.app.app_context():
-            flaskr.init_db()
-        yield client
-
-    os.close(db_fd)
-    os.unlink(flaskr.app.config['DATABASE'])
\ No newline at end of file

From 86b9b17c0b9dddc451eb89be4ead73f8fee5f608 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 3 Mar 2021 12:06:19 -0700
Subject: [PATCH 15/52] queueing

---
 README_local.md                               | 18 +++--
 parser/__init__.py                            |  2 +-
 parser/db.py                                  | 13 +++
 parser/main.py                                | 80 ++++++++++++-------
 parser/templates/upload_form.html             | 51 +++++-------
 .../tests/{test_parse.py => test_parser.py}   | 11 ++-
 6 files changed, 107 insertions(+), 68 deletions(-)
 rename parser/tests/{test_parse.py => test_parser.py} (61%)

diff --git a/README_local.md b/README_local.md
index 430b09a..27fac23 100644
--- a/README_local.md
+++ b/README_local.md
@@ -132,13 +132,21 @@ This script will output two files:
 - *names_extract* a guess for the extracted names (to be reviewed)
 
 # Running the app frontend
-app.py is a Flask-based frontend that provides a dedicated UI for uploading data sets and running the scripts with them.
+app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and running the scripts with them. It creates and manages a queue within a sqlite3 database (to be upgraded to Redis if performance needs call for it).
 
 Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
 
-To run an instance of the frontend locally:
+To run an instance of the frontend locally, from the project root directory initialize the db:
 `
-export FLASK_APP=app.py
+export FLASK_APP=parser
 export FLASK_ENV=development
-flask run
-`
\ No newline at end of file
+flask init-db
+`
+That will create a file named `parser.sqlite3` (the application database) in the `instance` directory. Then you should be able to do
+`flask run
+`
+and access the running application at [http://localhost:5000/](http://localhost:5000/)
+
+## Testing the app frontend
+
+`pytest` should run all the tests in the `tests` folder.
\ No newline at end of file
diff --git a/parser/__init__.py b/parser/__init__.py
index 24daca3..1091570 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -12,7 +12,7 @@ def create_app(test_config=None):
         SECRET_KEY=settings.SECRET_KEY,
         DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
         MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
-        UPLOAD_FOLDER=os.path.join(APP_ROOT, settings.UPLOAD_FOLDER)
+        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.UPLOAD_FOLDER)
     )
 
     from . import db
diff --git a/parser/db.py b/parser/db.py
index 0b4f188..fb126fd 100644
--- a/parser/db.py
+++ b/parser/db.py
@@ -15,6 +15,18 @@ def get_db():
 
     return g.db
 
+def query_db(query, args=(), one=False, insert=False):
+    import pdb; pdb.set_trace()
+    if insert:
+        conn = get_db()
+        conn.execute(query, args)
+        conn.commit()
+        return True
+    cur = get_db().execute(query, args)
+    rv = cur.fetchall()
+    cur.close()
+    return (rv[0] if rv else None) if one else rv
+
 
 def close_db(e=None):
     db = g.pop('db', None)
@@ -30,6 +42,7 @@ def init_db():
         db.executescript(f.read().decode('utf8'))
 
 
+
 @click.command('init-db')
 @with_appcontext
 def init_db_command():
diff --git a/parser/main.py b/parser/main.py
index e7b46ae..8f942dd 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -4,56 +4,76 @@
 import settings
 import sqlite3
 
-from flask import (Blueprint, flash, Flask, g, redirect, render_template, request,
-    send_from_directory, url_for)
-from parser.db import get_db
+from flask import (Blueprint, current_app, flash, Flask, g, redirect,
+    render_template, request, send_from_directory, url_for)
+from . import db
 from werkzeug.utils import secure_filename
 
 bp = Blueprint('main', __name__, url_prefix='')
 
 ALLOWED_EXTENSIONS = {'csv'}
-REQUIRED_HEADERS = {
-    'vec_file': ['names']
+UPLOAD_TYPES = {
+    'tblc_file': { 
+        'name': 'Text Banker Log Cleaning',
+        'required_headers': ['names']
+    },
+    'tblc_tmc_file': {
+        'name': 'Text Banker Log Cleaning (utilizing text message conversation)',
+        'required_headers': ['names']
+    },
+    'sccne_file': {
+        'name': 'SMS Conversation Categorization and Name Extraction',
+        'required_headers': []
+    },
+    'sms_agg_file': {
+        'name': 'SMS Aggregation',
+        'required_headers': []
+    }
 }
 
-# check filename extension on uploaded files
+
 def allowed_file(file):
     return ('.' in file.filename 
         and file.filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
         and file.content_type == 'text/csv')
 
-def check_headers(file, upload_type = 'vec_file'):
-    # check that the required headers are present in the file
-    headers = file.readline().decode('utf-8').split(',')
-    for header in REQUIRED_HEADERS[upload_type]:
+
+def check_headers(file, upload_type):
+    headers = file.readline().decode('utf-8').rstrip().split(',')
+    for header in UPLOAD_TYPES['required_headers']:
         if header not in headers:
             return False
     return True
 
-# For now, we're going to go ahead and run the script as soon as we get a file upload
-# Ultimately we want to use a queue to make sure we're doing these one at a time.
-def queue_job(type, message):
-    return
+
+def queue_job(file_path, job_type, email):
+    query = 'INSERT INTO jobs (file_path, job_type, email, status) VALUES (?, ?, ?, ?);'
+    db.query_db(query, (file_path, job_type, email, 'queued'), insert=True)
+    return True
+
 
 def process_job(job_id):
     return
 
+
 def complete_job(job_id):
     return
 
+
 def delete_file(filename):
     return
 
+
 # most of the business is here
 @bp.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
-        # make sure there's a REQUIRED_HEADERS entry for each file upload input
-        if not (REQUIRED_HEADERS.keys() & request.files.keys()):
+        if not (UPLOAD_TYPES.keys() & request.files.keys()):
             flash('No file', 'error')
             return redirect(request.url)
-        # iterate through the different file uploads
         for upload_type in request.files:
+            if upload_type not in UPLOAD_TYPES:
+                return redirect(request.url)
             file = request.files[upload_type]
             # If selected file is empty / without filename
             if file.filename == '':
@@ -61,20 +81,28 @@ def index():
                 return redirect(request.url)
             if not (file and allowed_file(file)):
                 flash('Invalid file type. Select a CSV to upload', 'error')
-                return render_template('upload_form.html')
-            if not check_headers(file, 'vec_file'):
-                flash('Invalid CSV headers. Make sure uploaded CSV has required columns.', 'error')
-                return render_template('upload_form.html')
+                return redirect(request.url)
+            # if not check_headers(file, upload_type):
+            #     flash('Invalid CSV headers. Check CSV for required columns.', 'error')
+            #     return redirect(request.url)
             # if it passes all the checks, save file and queue for processing
+
+            # TODO: Get email address!
+            # TODO: process file!
             filename = secure_filename(file.filename)
-            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
-            # TODO: queue for processing instead of spitting it back
-            return redirect(url_for('uploaded_file',
-                                    filename=filename))
+            file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
+            file.save(file_path)
+            result = queue_job(file_path, upload_type, 'file_test@example.com')
+            flash('Queued file {} for processing as {}. Check your email {} for results.'.format(filename, upload_type), 'info')
+            return redirect(request.url)
+
+            # return redirect(url_for('uploaded_file',
+            #                         filename=filename))
 
     # GET
     return render_template('upload_form.html')
 
+
 # returning uploaded files; get rid of this in prod
 @bp.route('/uploads/<filename>')
 def uploaded_file(filename):
@@ -84,5 +112,3 @@ def uploaded_file(filename):
     # i.e. uploads dir can't be inspected
 
 
-
-
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 42f12bc..22dbe9e 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -13,7 +13,9 @@
         a { text-decoration: none; color: royalblue; }
         a:hover { text-decoration: underline; color: purple; }
         a:visited { color: rebeccapurple; }
-        ul.flashes li {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
+        ul.flash li {font-size: 1.3em;}
+        ul.flashes li.error {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
+        ul.flashes li.info {list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee;}
         ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
         ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
@@ -28,11 +30,11 @@
 
     <main>
 
-    {% with messages = get_flashed_messages() %}
+    {% with messages = get_flashed_messages(with_categories=true) %}
       {% if messages %}
         <ul class=flashes>
-        {% for message in messages %}
-          <li>{{ message }}</li>
+        {% for category, message in messages %}
+          <li class="{{ category }}">{{ message }}</li>
         {% endfor %}
         </ul>
       {% endif %}
@@ -42,8 +44,9 @@
             <h1>Vote tripling SMS transcript parsing</h1>
 
             <ul class="menu">
-                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
                 <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+                <li><a href="#text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
+                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
                 <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
             </ul>
         </div>
@@ -51,43 +54,25 @@ <h1>Vote tripling SMS transcript parsing</h1>
         <div class="upload-enabled">
             <hr />
 
-            <h2 id='van_export_cleaning'>VAN Export Cleaning</h2>
-            <blockquote>I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
+            <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
 
-            <p>Upload a csv file with the following columns:</p>
-            <ul>
-                <li><strong>VANID:</strong> a unique ID for this row
-                <li><strong>ContactName:</strong> the name of the tripler
-                <li><strong>NoteText:</strong> free text possibly including names of tripling targets
-            </ul>
+            <blockquote> I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.</blockquote>
+
+            <p>Upload a csv containing a column 'names' containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="vec_file" name="vec_file" accept=".csv">
+                <input type=file id="tblc_file" name="tblc_file" accept=".csv">
                 <input type=submit value=Upload>
             </form>
 
-            <p>If successful, the output will include two files: 
-            <ul>
-                <li>A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
-                <ul>
-                    <li><strong>VANID:</strong> a unique identifier for the conversation</li>
-                    <li><strong>names_extract:</strong> the extracted names</li>
-                </ul>
+            <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
 
-                <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:</li>
-                <ul>
-                    <li><strong>VANID:</strong> a unique identifier for the conversation</li>
-                    <li><strong>ContactName:</strong> a unique identifier for the conversation</li>
-                    <li><strong>NoteText:</strong> free text possibly including names of tripling targets</li>
-                    <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)</li>
-                </ul>
-            </ul>
-        </  div>
+        </div>
 
         <div class="upload-disabled">
             <hr />
 
-            <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text message conversation)</h2>
+            <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</h2>
 
             <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
 
@@ -97,14 +82,14 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text messa
             This csv file must also contain column 'names' containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="tblc_file" name="tblc_file" accept=".csv">
+                <input type=file id="tblc_tmc_file" name="tblc_tmc_file" accept=".csv">
                 <input type=submit value=Upload>
             </form>
 
             <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
 
         </div>
-        
+
         <div class="upload-disabled">
             <hr />
             <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
diff --git a/parser/tests/test_parse.py b/parser/tests/test_parser.py
similarity index 61%
rename from parser/tests/test_parse.py
rename to parser/tests/test_parser.py
index 19020e3..d9d8b0b 100644
--- a/parser/tests/test_parse.py
+++ b/parser/tests/test_parser.py
@@ -3,7 +3,7 @@
 
 import pytest
 
-from . import parser
+import parser
 
 @pytest.fixture
 def client():
@@ -16,4 +16,11 @@ def client():
         yield client
 
     os.close(db_fd)
-    os.unlink(parser.app.config['DATABASE'])
\ No newline at end of file
+    os.unlink(parser.app.config['DATABASE'])
+
+
+# TODO
+
+# test posting wrong file extension to /
+# test posting files with bad headers to /
+# test posting right file with right headers to /

From c1266f9525f1eee335b351faafb34c19175ab674 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 3 Mar 2021 12:06:19 -0700
Subject: [PATCH 16/52] queueing

---
 README_local.md                               | 18 +++--
 parser/__init__.py                            |  2 +-
 parser/db.py                                  | 13 +++
 parser/main.py                                | 80 ++++++++++++-------
 parser/templates/upload_form.html             | 51 +++++-------
 .../tests/{test_parse.py => test_parser.py}   | 11 ++-
 6 files changed, 107 insertions(+), 68 deletions(-)
 rename parser/tests/{test_parse.py => test_parser.py} (61%)

diff --git a/README_local.md b/README_local.md
index 430b09a..27fac23 100644
--- a/README_local.md
+++ b/README_local.md
@@ -132,13 +132,21 @@ This script will output two files:
 - *names_extract* a guess for the extracted names (to be reviewed)
 
 # Running the app frontend
-app.py is a Flask-based frontend that provides a dedicated UI for uploading data sets and running the scripts with them.
+app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and running the scripts with them. It creates and manages a queue within a sqlite3 database (to be upgraded to Redis if performance needs call for it).
 
 Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
 
-To run an instance of the frontend locally:
+To run an instance of the frontend locally, from the project root directory initialize the db:
 `
-export FLASK_APP=app.py
+export FLASK_APP=parser
 export FLASK_ENV=development
-flask run
-`
\ No newline at end of file
+flask init-db
+`
+That will create a file named `parser.sqlite3` (the application database) in the `instance` directory. Then you should be able to do
+`flask run
+`
+and access the running application at [http://localhost:5000/](http://localhost:5000/)
+
+## Testing the app frontend
+
+`pytest` should run all the tests in the `tests` folder.
\ No newline at end of file
diff --git a/parser/__init__.py b/parser/__init__.py
index 24daca3..1091570 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -12,7 +12,7 @@ def create_app(test_config=None):
         SECRET_KEY=settings.SECRET_KEY,
         DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
         MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
-        UPLOAD_FOLDER=os.path.join(APP_ROOT, settings.UPLOAD_FOLDER)
+        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.UPLOAD_FOLDER)
     )
 
     from . import db
diff --git a/parser/db.py b/parser/db.py
index 0b4f188..fb126fd 100644
--- a/parser/db.py
+++ b/parser/db.py
@@ -15,6 +15,18 @@ def get_db():
 
     return g.db
 
+def query_db(query, args=(), one=False, insert=False):
+    import pdb; pdb.set_trace()
+    if insert:
+        conn = get_db()
+        conn.execute(query, args)
+        conn.commit()
+        return True
+    cur = get_db().execute(query, args)
+    rv = cur.fetchall()
+    cur.close()
+    return (rv[0] if rv else None) if one else rv
+
 
 def close_db(e=None):
     db = g.pop('db', None)
@@ -30,6 +42,7 @@ def init_db():
         db.executescript(f.read().decode('utf8'))
 
 
+
 @click.command('init-db')
 @with_appcontext
 def init_db_command():
diff --git a/parser/main.py b/parser/main.py
index e7b46ae..8f942dd 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -4,56 +4,76 @@
 import settings
 import sqlite3
 
-from flask import (Blueprint, flash, Flask, g, redirect, render_template, request,
-    send_from_directory, url_for)
-from parser.db import get_db
+from flask import (Blueprint, current_app, flash, Flask, g, redirect,
+    render_template, request, send_from_directory, url_for)
+from . import db
 from werkzeug.utils import secure_filename
 
 bp = Blueprint('main', __name__, url_prefix='')
 
 ALLOWED_EXTENSIONS = {'csv'}
-REQUIRED_HEADERS = {
-    'vec_file': ['names']
+UPLOAD_TYPES = {
+    'tblc_file': { 
+        'name': 'Text Banker Log Cleaning',
+        'required_headers': ['names']
+    },
+    'tblc_tmc_file': {
+        'name': 'Text Banker Log Cleaning (utilizing text message conversation)',
+        'required_headers': ['names']
+    },
+    'sccne_file': {
+        'name': 'SMS Conversation Categorization and Name Extraction',
+        'required_headers': []
+    },
+    'sms_agg_file': {
+        'name': 'SMS Aggregation',
+        'required_headers': []
+    }
 }
 
-# check filename extension on uploaded files
+
 def allowed_file(file):
     return ('.' in file.filename 
         and file.filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
         and file.content_type == 'text/csv')
 
-def check_headers(file, upload_type = 'vec_file'):
-    # check that the required headers are present in the file
-    headers = file.readline().decode('utf-8').split(',')
-    for header in REQUIRED_HEADERS[upload_type]:
+
+def check_headers(file, upload_type):
+    headers = file.readline().decode('utf-8').rstrip().split(',')
+    for header in UPLOAD_TYPES['required_headers']:
         if header not in headers:
             return False
     return True
 
-# For now, we're going to go ahead and run the script as soon as we get a file upload
-# Ultimately we want to use a queue to make sure we're doing these one at a time.
-def queue_job(type, message):
-    return
+
+def queue_job(file_path, job_type, email):
+    query = 'INSERT INTO jobs (file_path, job_type, email, status) VALUES (?, ?, ?, ?);'
+    db.query_db(query, (file_path, job_type, email, 'queued'), insert=True)
+    return True
+
 
 def process_job(job_id):
     return
 
+
 def complete_job(job_id):
     return
 
+
 def delete_file(filename):
     return
 
+
 # most of the business is here
 @bp.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
-        # make sure there's a REQUIRED_HEADERS entry for each file upload input
-        if not (REQUIRED_HEADERS.keys() & request.files.keys()):
+        if not (UPLOAD_TYPES.keys() & request.files.keys()):
             flash('No file', 'error')
             return redirect(request.url)
-        # iterate through the different file uploads
         for upload_type in request.files:
+            if upload_type not in UPLOAD_TYPES:
+                return redirect(request.url)
             file = request.files[upload_type]
             # If selected file is empty / without filename
             if file.filename == '':
@@ -61,20 +81,28 @@ def index():
                 return redirect(request.url)
             if not (file and allowed_file(file)):
                 flash('Invalid file type. Select a CSV to upload', 'error')
-                return render_template('upload_form.html')
-            if not check_headers(file, 'vec_file'):
-                flash('Invalid CSV headers. Make sure uploaded CSV has required columns.', 'error')
-                return render_template('upload_form.html')
+                return redirect(request.url)
+            # if not check_headers(file, upload_type):
+            #     flash('Invalid CSV headers. Check CSV for required columns.', 'error')
+            #     return redirect(request.url)
             # if it passes all the checks, save file and queue for processing
+
+            # TODO: Get email address!
+            # TODO: process file!
             filename = secure_filename(file.filename)
-            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
-            # TODO: queue for processing instead of spitting it back
-            return redirect(url_for('uploaded_file',
-                                    filename=filename))
+            file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
+            file.save(file_path)
+            result = queue_job(file_path, upload_type, 'file_test@example.com')
+            flash('Queued file {} for processing as {}. Check your email {} for results.'.format(filename, upload_type), 'info')
+            return redirect(request.url)
+
+            # return redirect(url_for('uploaded_file',
+            #                         filename=filename))
 
     # GET
     return render_template('upload_form.html')
 
+
 # returning uploaded files; get rid of this in prod
 @bp.route('/uploads/<filename>')
 def uploaded_file(filename):
@@ -84,5 +112,3 @@ def uploaded_file(filename):
     # i.e. uploads dir can't be inspected
 
 
-
-
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 42f12bc..22dbe9e 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -13,7 +13,9 @@
         a { text-decoration: none; color: royalblue; }
         a:hover { text-decoration: underline; color: purple; }
         a:visited { color: rebeccapurple; }
-        ul.flashes li {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
+        ul.flash li {font-size: 1.3em;}
+        ul.flashes li.error {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
+        ul.flashes li.info {list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee;}
         ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
         ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
@@ -28,11 +30,11 @@
 
     <main>
 
-    {% with messages = get_flashed_messages() %}
+    {% with messages = get_flashed_messages(with_categories=true) %}
       {% if messages %}
         <ul class=flashes>
-        {% for message in messages %}
-          <li>{{ message }}</li>
+        {% for category, message in messages %}
+          <li class="{{ category }}">{{ message }}</li>
         {% endfor %}
         </ul>
       {% endif %}
@@ -42,8 +44,9 @@
             <h1>Vote tripling SMS transcript parsing</h1>
 
             <ul class="menu">
-                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
                 <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+                <li><a href="#text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
+                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
                 <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
             </ul>
         </div>
@@ -51,43 +54,25 @@ <h1>Vote tripling SMS transcript parsing</h1>
         <div class="upload-enabled">
             <hr />
 
-            <h2 id='van_export_cleaning'>VAN Export Cleaning</h2>
-            <blockquote>I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
+            <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
 
-            <p>Upload a csv file with the following columns:</p>
-            <ul>
-                <li><strong>VANID:</strong> a unique ID for this row
-                <li><strong>ContactName:</strong> the name of the tripler
-                <li><strong>NoteText:</strong> free text possibly including names of tripling targets
-            </ul>
+            <blockquote> I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.</blockquote>
+
+            <p>Upload a csv containing a column 'names' containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="vec_file" name="vec_file" accept=".csv">
+                <input type=file id="tblc_file" name="tblc_file" accept=".csv">
                 <input type=submit value=Upload>
             </form>
 
-            <p>If successful, the output will include two files: 
-            <ul>
-                <li>A file of triplers called `van_cleaned.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
-                <ul>
-                    <li><strong>VANID:</strong> a unique identifier for the conversation</li>
-                    <li><strong>names_extract:</strong> the extracted names</li>
-                </ul>
+            <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
 
-                <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:</li>
-                <ul>
-                    <li><strong>VANID:</strong> a unique identifier for the conversation</li>
-                    <li><strong>ContactName:</strong> a unique identifier for the conversation</li>
-                    <li><strong>NoteText:</strong> free text possibly including names of tripling targets</li>
-                    <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)</li>
-                </ul>
-            </ul>
-        </  div>
+        </div>
 
         <div class="upload-disabled">
             <hr />
 
-            <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text message conversation)</h2>
+            <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</h2>
 
             <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
 
@@ -97,14 +82,14 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning (utilizing text messa
             This csv file must also contain column 'names' containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="tblc_file" name="tblc_file" accept=".csv">
+                <input type=file id="tblc_tmc_file" name="tblc_tmc_file" accept=".csv">
                 <input type=submit value=Upload>
             </form>
 
             <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
 
         </div>
-        
+
         <div class="upload-disabled">
             <hr />
             <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
diff --git a/parser/tests/test_parse.py b/parser/tests/test_parser.py
similarity index 61%
rename from parser/tests/test_parse.py
rename to parser/tests/test_parser.py
index 19020e3..d9d8b0b 100644
--- a/parser/tests/test_parse.py
+++ b/parser/tests/test_parser.py
@@ -3,7 +3,7 @@
 
 import pytest
 
-from . import parser
+import parser
 
 @pytest.fixture
 def client():
@@ -16,4 +16,11 @@ def client():
         yield client
 
     os.close(db_fd)
-    os.unlink(parser.app.config['DATABASE'])
\ No newline at end of file
+    os.unlink(parser.app.config['DATABASE'])
+
+
+# TODO
+
+# test posting wrong file extension to /
+# test posting files with bad headers to /
+# test posting right file with right headers to /

From a88e60bdecd88c1550cd5122aae99d71aacbd1cf Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 5 Mar 2021 18:08:30 -0700
Subject: [PATCH 17/52] updated to start processing jobs

---
 parser/__init__.py                |  3 +-
 parser/db.py                      |  6 +--
 parser/main.py                    | 87 ++++++++++++++++++++++---------
 parser/schema.sql                 |  3 +-
 parser/templates/upload_form.html | 23 ++++++--
 5 files changed, 85 insertions(+), 37 deletions(-)

diff --git a/parser/__init__.py b/parser/__init__.py
index 1091570..8cd55b0 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -12,7 +12,8 @@ def create_app(test_config=None):
         SECRET_KEY=settings.SECRET_KEY,
         DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
         MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
-        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.UPLOAD_FOLDER)
+        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.UPLOAD_FOLDER),
+        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.RESULTS_FOLDER)
     )
 
     from . import db
diff --git a/parser/db.py b/parser/db.py
index fb126fd..8d4df7a 100644
--- a/parser/db.py
+++ b/parser/db.py
@@ -15,9 +15,8 @@ def get_db():
 
     return g.db
 
-def query_db(query, args=(), one=False, insert=False):
-    import pdb; pdb.set_trace()
-    if insert:
+def query_db(query, args=(), one=False, write=False):
+    if write:
         conn = get_db()
         conn.execute(query, args)
         conn.commit()
@@ -42,7 +41,6 @@ def init_db():
         db.executescript(f.read().decode('utf8'))
 
 
-
 @click.command('init-db')
 @with_appcontext
 def init_db_command():
diff --git a/parser/main.py b/parser/main.py
index 8f942dd..b1f76c8 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -3,6 +3,7 @@
 import os
 import settings
 import sqlite3
+import subprocess
 
 from flask import (Blueprint, current_app, flash, Flask, g, redirect,
     render_template, request, send_from_directory, url_for)
@@ -47,22 +48,54 @@ def check_headers(file, upload_type):
 
 
 def queue_job(file_path, job_type, email):
-    query = 'INSERT INTO jobs (file_path, job_type, email, status) VALUES (?, ?, ?, ?);'
-    db.query_db(query, (file_path, job_type, email, 'queued'), insert=True)
+    query = 'INSERT INTO jobs (input_file, job_type, email, status) VALUES (?, ?, ?, ?);'
+    db.query_db(query, (file_path, job_type, email, 'queued'), write=True)
     return True
 
 
-def process_job(job_id):
-    return
-
-
-def complete_job(job_id):
-    return
-
-
-def delete_file(filename):
-    return
-
+def process_job():
+    # if there are any queued jobs, process the oldest one
+    query = 'SELECT id, input_file, job_type FROM jobs WHERE status = ? ORDER BY created_at ASC;'
+    jobs = db.query_db(query, ('queued'), one=True)
+    if not jobs:
+        print("No jobs to process")
+        return False
+    job_id = job['id']
+    input_file = job['input_file']
+    job_type = job['job_type']
+    print("Processing job {} file {} type {}".format(job_id, input_file, job_type))
+    if job_type == 'tblc_file':
+        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/name_cleaning.py'
+        args = '-i {}'.format(input_file)
+    elif job_type == 'tblc_tmc_file':
+        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/name_cleaning_with_responses.py'
+        args = '-d {}'.format(input_file)
+    elif job_type == 'sccne_file':
+        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/annotate_conversations.py'
+        args = '-d {}'.(input_file)
+    # elif job_type == 'sms_agg_file':
+        # this script is to be ported to Python
+    job_run = subprocess.run([cmd, args])
+    # TODO: capture any errors and return whether job was successful
+    # TODO: re-name the results file to a random string
+    # TODO: Update db entry with results file path and status
+    status = 'success'
+
+    return status
+
+
+def email_results():
+    pass
+
+
+def cleanup_files(filename):
+    # delete files that are more than 48 hours old
+    pass
+
+
+def cleanup_jobs(interval):
+    # delete jobs that are more than `interval` old, to keep the db from getting too big
+    pass
 
 # most of the business is here
 @bp.route('/', methods=['GET', 'POST'])
@@ -85,30 +118,32 @@ def index():
             # if not check_headers(file, upload_type):
             #     flash('Invalid CSV headers. Check CSV for required columns.', 'error')
             #     return redirect(request.url)
-            # if it passes all the checks, save file and queue for processing
 
-            # TODO: Get email address!
-            # TODO: process file!
+            # if it passes all the checks, save file and queue for processing
             filename = secure_filename(file.filename)
             file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
             file.save(file_path)
-            result = queue_job(file_path, upload_type, 'file_test@example.com')
-            flash('Queued file {} for processing as {}. Check your email {} for results.'.format(filename, upload_type), 'info')
-            return redirect(request.url)
-
-            # return redirect(url_for('uploaded_file',
-            #                         filename=filename))
+            email_name = '{}_email'.format(upload_type.split('_')[0])
+            email = request.form[email_name]
+            result = queue_job(file_path, upload_type, email)
+            msg = ('Queued file {} for processing as {}. Check your email {} '
+                   'in a few minutes for results.').format(
+                       filename,
+                       UPLOAD_TYPES[upload_type]['name'],
+                       email)
+            flash(queue_message, 'info')
 
+            return redirect(request.url)
     # GET
     return render_template('upload_form.html')
 
 
-# returning uploaded files; get rid of this in prod
-@bp.route('/uploads/<filename>')
+# returning results files
+@bp.route('/results/<filename>')
 def uploaded_file(filename):
-    return send_from_directory(app.config['UPLOAD_FOLDER'],
+    return send_from_directory(current_app.config['RESULTS_FOLDER'],
                                filename)
     # TODO: documentation on setting file directory permissions
-    # i.e. uploads dir can't be inspected
+    # i.e. results dir can't be inspected
 
 
diff --git a/parser/schema.sql b/parser/schema.sql
index dd99695..3f8854b 100644
--- a/parser/schema.sql
+++ b/parser/schema.sql
@@ -3,7 +3,8 @@ DROP TABLE IF EXISTS jobs;
 CREATE TABLE jobs (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  file_path TEXT NOT NULL,
+  input_file TEXT NOT NULL,
+  result_file TEXT,
   job_type TEXT NOT NULL,
   email TEXT NOT NULL,
   status TEXT NOT NULL
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 22dbe9e..b050d69 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -13,6 +13,7 @@
         a { text-decoration: none; color: royalblue; }
         a:hover { text-decoration: underline; color: purple; }
         a:visited { color: rebeccapurple; }
+        form div { padding: .3em; margin: .3em .5em;}
         ul.flash li {font-size: 1.3em;}
         ul.flashes li.error {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
         ul.flashes li.info {list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee;}
@@ -20,10 +21,14 @@
         ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
         input, button { font-size: .9em; height: 2em; }
-        blockquote { display: block;  margin-left: 0; padding: 1em; background-color: rgba(255,255,255,.8); }
+        blockquote, form { display: block;  margin-left: 0; padding: 1em; color: #444; background-color: rgba(255,255,255,.8); max-width: 90%; }
         div.upload-disabled { opacity: .3; }
-        div.upload-disabled input {display: none;}
+        div.upload-disabled form {display: none;}
     </style>
+
+    <script type="text/javascript">
+        // TODO: validation JS for forms
+    </script>
 </head>
 
 <body>
@@ -58,11 +63,19 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
 
             <blockquote> I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.</blockquote>
 
-            <p>Upload a csv containing a column 'names' containing the names logged by a text banker.</p>
+            <p>First, enter your email address to receive a link to the output file.</p>
+            <p>Then, upload a csv containing a column 'names' containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="tblc_file" name="tblc_file" accept=".csv">
-                <input type=submit value=Upload>
+                <div>
+                    <label for="tblc_email">Your email:</label>
+                    <input type="email" id="tblc_email" name="tblc_email" size="30">
+                </div>
+                <div>
+                    <label for="tblc_file">Your CSV:</label>
+                    <input type="file" id="tblc_file" name="tblc_file" accept=".csv">
+                    <input type=submit value=Upload>
+                </div>
             </form>
 
             <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>

From 7c06ec44d5eee1cacbb6a1e64ed119cdc7d914f6 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 5 Mar 2021 18:08:30 -0700
Subject: [PATCH 18/52] updated to start processing jobs

---
 parser/__init__.py                |  3 +-
 parser/db.py                      |  6 +--
 parser/main.py                    | 87 ++++++++++++++++++++++---------
 parser/schema.sql                 |  3 +-
 parser/templates/upload_form.html | 23 ++++++--
 5 files changed, 85 insertions(+), 37 deletions(-)

diff --git a/parser/__init__.py b/parser/__init__.py
index 1091570..8cd55b0 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -12,7 +12,8 @@ def create_app(test_config=None):
         SECRET_KEY=settings.SECRET_KEY,
         DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
         MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
-        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.UPLOAD_FOLDER)
+        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.UPLOAD_FOLDER),
+        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.RESULTS_FOLDER)
     )
 
     from . import db
diff --git a/parser/db.py b/parser/db.py
index fb126fd..8d4df7a 100644
--- a/parser/db.py
+++ b/parser/db.py
@@ -15,9 +15,8 @@ def get_db():
 
     return g.db
 
-def query_db(query, args=(), one=False, insert=False):
-    import pdb; pdb.set_trace()
-    if insert:
+def query_db(query, args=(), one=False, write=False):
+    if write:
         conn = get_db()
         conn.execute(query, args)
         conn.commit()
@@ -42,7 +41,6 @@ def init_db():
         db.executescript(f.read().decode('utf8'))
 
 
-
 @click.command('init-db')
 @with_appcontext
 def init_db_command():
diff --git a/parser/main.py b/parser/main.py
index 8f942dd..b1f76c8 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -3,6 +3,7 @@
 import os
 import settings
 import sqlite3
+import subprocess
 
 from flask import (Blueprint, current_app, flash, Flask, g, redirect,
     render_template, request, send_from_directory, url_for)
@@ -47,22 +48,54 @@ def check_headers(file, upload_type):
 
 
 def queue_job(file_path, job_type, email):
-    query = 'INSERT INTO jobs (file_path, job_type, email, status) VALUES (?, ?, ?, ?);'
-    db.query_db(query, (file_path, job_type, email, 'queued'), insert=True)
+    query = 'INSERT INTO jobs (input_file, job_type, email, status) VALUES (?, ?, ?, ?);'
+    db.query_db(query, (file_path, job_type, email, 'queued'), write=True)
     return True
 
 
-def process_job(job_id):
-    return
-
-
-def complete_job(job_id):
-    return
-
-
-def delete_file(filename):
-    return
-
+def process_job():
+    # if there are any queued jobs, process the oldest one
+    query = 'SELECT id, input_file, job_type FROM jobs WHERE status = ? ORDER BY created_at ASC;'
+    jobs = db.query_db(query, ('queued'), one=True)
+    if not jobs:
+        print("No jobs to process")
+        return False
+    job_id = job['id']
+    input_file = job['input_file']
+    job_type = job['job_type']
+    print("Processing job {} file {} type {}".format(job_id, input_file, job_type))
+    if job_type == 'tblc_file':
+        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/name_cleaning.py'
+        args = '-i {}'.format(input_file)
+    elif job_type == 'tblc_tmc_file':
+        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/name_cleaning_with_responses.py'
+        args = '-d {}'.format(input_file)
+    elif job_type == 'sccne_file':
+        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/annotate_conversations.py'
+        args = '-d {}'.(input_file)
+    # elif job_type == 'sms_agg_file':
+        # this script is to be ported to Python
+    job_run = subprocess.run([cmd, args])
+    # TODO: capture any errors and return whether job was successful
+    # TODO: re-name the results file to a random string
+    # TODO: Update db entry with results file path and status
+    status = 'success'
+
+    return status
+
+
+def email_results():
+    pass
+
+
+def cleanup_files(filename):
+    # delete files that are more than 48 hours old
+    pass
+
+
+def cleanup_jobs(interval):
+    # delete jobs that are more than `interval` old, to keep the db from getting too big
+    pass
 
 # most of the business is here
 @bp.route('/', methods=['GET', 'POST'])
@@ -85,30 +118,32 @@ def index():
             # if not check_headers(file, upload_type):
             #     flash('Invalid CSV headers. Check CSV for required columns.', 'error')
             #     return redirect(request.url)
-            # if it passes all the checks, save file and queue for processing
 
-            # TODO: Get email address!
-            # TODO: process file!
+            # if it passes all the checks, save file and queue for processing
             filename = secure_filename(file.filename)
             file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
             file.save(file_path)
-            result = queue_job(file_path, upload_type, 'file_test@example.com')
-            flash('Queued file {} for processing as {}. Check your email {} for results.'.format(filename, upload_type), 'info')
-            return redirect(request.url)
-
-            # return redirect(url_for('uploaded_file',
-            #                         filename=filename))
+            email_name = '{}_email'.format(upload_type.split('_')[0])
+            email = request.form[email_name]
+            result = queue_job(file_path, upload_type, email)
+            msg = ('Queued file {} for processing as {}. Check your email {} '
+                   'in a few minutes for results.').format(
+                       filename,
+                       UPLOAD_TYPES[upload_type]['name'],
+                       email)
+            flash(queue_message, 'info')
 
+            return redirect(request.url)
     # GET
     return render_template('upload_form.html')
 
 
-# returning uploaded files; get rid of this in prod
-@bp.route('/uploads/<filename>')
+# returning results files
+@bp.route('/results/<filename>')
 def uploaded_file(filename):
-    return send_from_directory(app.config['UPLOAD_FOLDER'],
+    return send_from_directory(current_app.config['RESULTS_FOLDER'],
                                filename)
     # TODO: documentation on setting file directory permissions
-    # i.e. uploads dir can't be inspected
+    # i.e. results dir can't be inspected
 
 
diff --git a/parser/schema.sql b/parser/schema.sql
index dd99695..3f8854b 100644
--- a/parser/schema.sql
+++ b/parser/schema.sql
@@ -3,7 +3,8 @@ DROP TABLE IF EXISTS jobs;
 CREATE TABLE jobs (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  file_path TEXT NOT NULL,
+  input_file TEXT NOT NULL,
+  result_file TEXT,
   job_type TEXT NOT NULL,
   email TEXT NOT NULL,
   status TEXT NOT NULL
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 22dbe9e..b050d69 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -13,6 +13,7 @@
         a { text-decoration: none; color: royalblue; }
         a:hover { text-decoration: underline; color: purple; }
         a:visited { color: rebeccapurple; }
+        form div { padding: .3em; margin: .3em .5em;}
         ul.flash li {font-size: 1.3em;}
         ul.flashes li.error {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
         ul.flashes li.info {list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee;}
@@ -20,10 +21,14 @@
         ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
         input, button { font-size: .9em; height: 2em; }
-        blockquote { display: block;  margin-left: 0; padding: 1em; background-color: rgba(255,255,255,.8); }
+        blockquote, form { display: block;  margin-left: 0; padding: 1em; color: #444; background-color: rgba(255,255,255,.8); max-width: 90%; }
         div.upload-disabled { opacity: .3; }
-        div.upload-disabled input {display: none;}
+        div.upload-disabled form {display: none;}
     </style>
+
+    <script type="text/javascript">
+        // TODO: validation JS for forms
+    </script>
 </head>
 
 <body>
@@ -58,11 +63,19 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
 
             <blockquote> I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.</blockquote>
 
-            <p>Upload a csv containing a column 'names' containing the names logged by a text banker.</p>
+            <p>First, enter your email address to receive a link to the output file.</p>
+            <p>Then, upload a csv containing a column 'names' containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="tblc_file" name="tblc_file" accept=".csv">
-                <input type=submit value=Upload>
+                <div>
+                    <label for="tblc_email">Your email:</label>
+                    <input type="email" id="tblc_email" name="tblc_email" size="30">
+                </div>
+                <div>
+                    <label for="tblc_file">Your CSV:</label>
+                    <input type="file" id="tblc_file" name="tblc_file" accept=".csv">
+                    <input type=submit value=Upload>
+                </div>
             </form>
 
             <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>

From 6da4c0f276327f030adae5fc7e5d33ebe61cc895 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 10 Mar 2021 13:46:13 -0700
Subject: [PATCH 19/52] simplest case works without queueing

---
 parser/__init__.py                |  7 ++-
 parser/main.py                    | 91 +++++++++++++++++++++++--------
 parser/templates/upload_form.html |  8 ++-
 requirements.txt                  |  1 +
 4 files changed, 78 insertions(+), 29 deletions(-)

diff --git a/parser/__init__.py b/parser/__init__.py
index 8cd55b0..8cc4947 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -12,8 +12,11 @@ def create_app(test_config=None):
         SECRET_KEY=settings.SECRET_KEY,
         DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
         MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
-        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.UPLOAD_FOLDER),
-        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.RESULTS_FOLDER)
+        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Input_data'),
+        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_data'),
+        SCRIPTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Code')
+        # CELERY_BROKER_URL=,
+        # CELERY_RESULT_BACKEND=
     )
 
     from . import db
diff --git a/parser/main.py b/parser/main.py
index b1f76c8..7e341d9 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -1,12 +1,15 @@
-import functools
 import csv
+import datetime
+import functools
 import os
+import random
 import settings
 import sqlite3
+import string
 import subprocess
 
-from flask import (Blueprint, current_app, flash, Flask, g, redirect,
-    render_template, request, send_from_directory, url_for)
+from flask import (Blueprint, current_app, flash, Flask, g, Markup,
+    redirect, render_template, request, send_from_directory, url_for)
 from . import db
 from werkzeug.utils import secure_filename
 
@@ -54,34 +57,63 @@ def queue_job(file_path, job_type, email):
 
 
 def process_job():
+    """
+        TODO: Replace with a celery queue and workers.
+    """
+
     # if there are any queued jobs, process the oldest one
     query = 'SELECT id, input_file, job_type FROM jobs WHERE status = ? ORDER BY created_at ASC;'
-    jobs = db.query_db(query, ('queued'), one=True)
-    if not jobs:
+    job = db.query_db(query, ('queued',), one=True)
+    if not job:
         print("No jobs to process")
         return False
     job_id = job['id']
     input_file = job['input_file']
     job_type = job['job_type']
     print("Processing job {} file {} type {}".format(job_id, input_file, job_type))
+    
+    status = 'success'
+    processing_query = 'UPDATE jobs SET status = ? WHERE id = ?;'
+    update = db.query_db(processing_query, ('processing', job_id), write=True)
+
+    # It seems silly to run these via the command line but it's for ease of 
+    # maintenance by non-engineers.
+    scripts_folder = current_app.config['SCRIPTS_FOLDER']
+    scripts_home_dir = os.path.dirname(scripts_folder)
+    print('shd', scripts_home_dir)
+    output_file = '{}-{}.csv'.format(''.join(random.SystemRandom().choice(
+        string.ascii_lowercase + string.digits) for _ in range(17)), 
+        datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
     if job_type == 'tblc_file':
-        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/name_cleaning.py'
-        args = '-i {}'.format(input_file)
+        cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file)
     elif job_type == 'tblc_tmc_file':
-        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/name_cleaning_with_responses.py'
-        args = '-d {}'.format(input_file)
+        cmd = 'python {}/name_cleaning_with_responses.py'.format(scripts_folder)
+        args = ['-d {}'.format(input_file)]
     elif job_type == 'sccne_file':
-        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/annotate_conversations.py'
-        args = '-d {}'.(input_file)
+        cmd = 'python {}/annotate_conversations.py'.format(scripts_folder)
+        args = ['-d {}'.format(input_file)]
     # elif job_type == 'sms_agg_file':
         # this script is to be ported to Python
-    job_run = subprocess.run([cmd, args])
-    # TODO: capture any errors and return whether job was successful
-    # TODO: re-name the results file to a random string
-    # TODO: Update db entry with results file path and status
-    status = 'success'
-
-    return status
+    else:
+        print("Unknown job type {}".format(job_type))
+    print("cmd", cmd)
+    job_run = subprocess.run(cmd, stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT, shell=True)
+    if job_run.returncode: # exit code of 0 is success, 1 is generic error
+        status = 'error'
+        err_log = job_run.stdout.decode()
+        print("ERROR: Could not process job {} file {} type {}".format(job_id, input_file, job_type))
+        print(err_log)
+    else:
+        print("SUCCESS")
+    result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], 
+        output_file) if status == 'success' else None
+    done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
+    update = db.query_db(done_query, (status, result_file, job_id), write=True)
+    if status == 'success':
+        return output_file
+    return False
 
 
 def email_results():
@@ -126,13 +158,24 @@ def index():
             email_name = '{}_email'.format(upload_type.split('_')[0])
             email = request.form[email_name]
             result = queue_job(file_path, upload_type, email)
-            msg = ('Queued file {} for processing as {}. Check your email {} '
-                   'in a few minutes for results.').format(
-                       filename,
-                       UPLOAD_TYPES[upload_type]['name'],
-                       email)
-            flash(queue_message, 'info')
 
+            # This message is currently a lie, we're not really queueing jobs now
+            # except for very briefly.
+            # queue_msg = ('Queued file {} for processing as {}. Check your email {} '
+            #        'in a few minutes for results.').format(
+            #            filename,
+            #            UPLOAD_TYPES[upload_type]['name'],
+            #            email)
+            # queue_msg = ('Processing file {} as {}'.format(
+            #                 filename, UPLOAD_TYPES[upload_type]['name']))
+            # flash(queue_msg, 'info')
+            success = process_job()
+            if success:
+                outcome_msg = Markup(
+                    '<a href="/results/{}">Download results</a>'.format(success))
+            else:
+                outcome_msg = 'Error processing file'
+            flash(outcome_msg, 'info')
             return redirect(request.url)
     # GET
     return render_template('upload_form.html')
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index b050d69..0749b61 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -17,9 +17,11 @@
         ul.flash li {font-size: 1.3em;}
         ul.flashes li.error {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
         ul.flashes li.info {list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee;}
+        ul.flashes li.info a {text-decoration: underline; color: white;}
         ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
         ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
+        ul.menu li a.disabled { opacity: 0.2; }
         input, button { font-size: .9em; height: 2em; }
         blockquote, form { display: block;  margin-left: 0; padding: 1em; color: #444; background-color: rgba(255,255,255,.8); max-width: 90%; }
         div.upload-disabled { opacity: .3; }
@@ -50,9 +52,9 @@ <h1>Vote tripling SMS transcript parsing</h1>
 
             <ul class="menu">
                 <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
-                <li><a href="#text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
-                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
-                <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
+                <li><a href="#text_banker_log_cleaning_with_conversation" class="disabled">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
+                <li><a href="#van_export_cleaning" class="disabled">VAN export cleaning</a></li>
+                <li><a href="#sms_convo_cat_name_extract" class="disabled">SMS Conversation Categorization and Name Extraction</a></li>
             </ul>
         </div>
 
diff --git a/requirements.txt b/requirements.txt
index 496c04e..94b8308 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+celery==5.0.5
 coverage==5.4
 Flask==1.1.2
 matplotlib==3.3.3

From 60cec14144cc63f83e8a7a65156fd8bc2ebb09f4 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 10 Mar 2021 13:46:13 -0700
Subject: [PATCH 20/52] simplest case works without queueing

---
 parser/__init__.py                |  7 ++-
 parser/main.py                    | 91 +++++++++++++++++++++++--------
 parser/templates/upload_form.html |  8 ++-
 requirements.txt                  |  1 +
 4 files changed, 78 insertions(+), 29 deletions(-)

diff --git a/parser/__init__.py b/parser/__init__.py
index 8cd55b0..8cc4947 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -12,8 +12,11 @@ def create_app(test_config=None):
         SECRET_KEY=settings.SECRET_KEY,
         DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
         MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
-        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.UPLOAD_FOLDER),
-        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), settings.RESULTS_FOLDER)
+        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Input_data'),
+        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_data'),
+        SCRIPTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Code')
+        # CELERY_BROKER_URL=,
+        # CELERY_RESULT_BACKEND=
     )
 
     from . import db
diff --git a/parser/main.py b/parser/main.py
index b1f76c8..7e341d9 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -1,12 +1,15 @@
-import functools
 import csv
+import datetime
+import functools
 import os
+import random
 import settings
 import sqlite3
+import string
 import subprocess
 
-from flask import (Blueprint, current_app, flash, Flask, g, redirect,
-    render_template, request, send_from_directory, url_for)
+from flask import (Blueprint, current_app, flash, Flask, g, Markup,
+    redirect, render_template, request, send_from_directory, url_for)
 from . import db
 from werkzeug.utils import secure_filename
 
@@ -54,34 +57,63 @@ def queue_job(file_path, job_type, email):
 
 
 def process_job():
+    """
+        TODO: Replace with a celery queue and workers.
+    """
+
     # if there are any queued jobs, process the oldest one
     query = 'SELECT id, input_file, job_type FROM jobs WHERE status = ? ORDER BY created_at ASC;'
-    jobs = db.query_db(query, ('queued'), one=True)
-    if not jobs:
+    job = db.query_db(query, ('queued',), one=True)
+    if not job:
         print("No jobs to process")
         return False
     job_id = job['id']
     input_file = job['input_file']
     job_type = job['job_type']
     print("Processing job {} file {} type {}".format(job_id, input_file, job_type))
+    
+    status = 'success'
+    processing_query = 'UPDATE jobs SET status = ? WHERE id = ?;'
+    update = db.query_db(processing_query, ('processing', job_id), write=True)
+
+    # It seems silly to run these via the command line but it's for ease of 
+    # maintenance by non-engineers.
+    scripts_folder = current_app.config['SCRIPTS_FOLDER']
+    scripts_home_dir = os.path.dirname(scripts_folder)
+    print('shd', scripts_home_dir)
+    output_file = '{}-{}.csv'.format(''.join(random.SystemRandom().choice(
+        string.ascii_lowercase + string.digits) for _ in range(17)), 
+        datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
     if job_type == 'tblc_file':
-        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/name_cleaning.py'
-        args = '-i {}'.format(input_file)
+        cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file)
     elif job_type == 'tblc_tmc_file':
-        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/name_cleaning_with_responses.py'
-        args = '-d {}'.format(input_file)
+        cmd = 'python {}/name_cleaning_with_responses.py'.format(scripts_folder)
+        args = ['-d {}'.format(input_file)]
     elif job_type == 'sccne_file':
-        cmd = 'python ../Projects/NLP/SMS_Annotation/Code/annotate_conversations.py'
-        args = '-d {}'.(input_file)
+        cmd = 'python {}/annotate_conversations.py'.format(scripts_folder)
+        args = ['-d {}'.format(input_file)]
     # elif job_type == 'sms_agg_file':
         # this script is to be ported to Python
-    job_run = subprocess.run([cmd, args])
-    # TODO: capture any errors and return whether job was successful
-    # TODO: re-name the results file to a random string
-    # TODO: Update db entry with results file path and status
-    status = 'success'
-
-    return status
+    else:
+        print("Unknown job type {}".format(job_type))
+    print("cmd", cmd)
+    job_run = subprocess.run(cmd, stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT, shell=True)
+    if job_run.returncode: # exit code of 0 is success, 1 is generic error
+        status = 'error'
+        err_log = job_run.stdout.decode()
+        print("ERROR: Could not process job {} file {} type {}".format(job_id, input_file, job_type))
+        print(err_log)
+    else:
+        print("SUCCESS")
+    result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], 
+        output_file) if status == 'success' else None
+    done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
+    update = db.query_db(done_query, (status, result_file, job_id), write=True)
+    if status == 'success':
+        return output_file
+    return False
 
 
 def email_results():
@@ -126,13 +158,24 @@ def index():
             email_name = '{}_email'.format(upload_type.split('_')[0])
             email = request.form[email_name]
             result = queue_job(file_path, upload_type, email)
-            msg = ('Queued file {} for processing as {}. Check your email {} '
-                   'in a few minutes for results.').format(
-                       filename,
-                       UPLOAD_TYPES[upload_type]['name'],
-                       email)
-            flash(queue_message, 'info')
 
+            # This message is currently a lie, we're not really queueing jobs now
+            # except for very briefly.
+            # queue_msg = ('Queued file {} for processing as {}. Check your email {} '
+            #        'in a few minutes for results.').format(
+            #            filename,
+            #            UPLOAD_TYPES[upload_type]['name'],
+            #            email)
+            # queue_msg = ('Processing file {} as {}'.format(
+            #                 filename, UPLOAD_TYPES[upload_type]['name']))
+            # flash(queue_msg, 'info')
+            success = process_job()
+            if success:
+                outcome_msg = Markup(
+                    '<a href="/results/{}">Download results</a>'.format(success))
+            else:
+                outcome_msg = 'Error processing file'
+            flash(outcome_msg, 'info')
             return redirect(request.url)
     # GET
     return render_template('upload_form.html')
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index b050d69..0749b61 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -17,9 +17,11 @@
         ul.flash li {font-size: 1.3em;}
         ul.flashes li.error {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
         ul.flashes li.info {list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee;}
+        ul.flashes li.info a {text-decoration: underline; color: white;}
         ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
         ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
+        ul.menu li a.disabled { opacity: 0.2; }
         input, button { font-size: .9em; height: 2em; }
         blockquote, form { display: block;  margin-left: 0; padding: 1em; color: #444; background-color: rgba(255,255,255,.8); max-width: 90%; }
         div.upload-disabled { opacity: .3; }
@@ -50,9 +52,9 @@ <h1>Vote tripling SMS transcript parsing</h1>
 
             <ul class="menu">
                 <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
-                <li><a href="#text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
-                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
-                <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
+                <li><a href="#text_banker_log_cleaning_with_conversation" class="disabled">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
+                <li><a href="#van_export_cleaning" class="disabled">VAN export cleaning</a></li>
+                <li><a href="#sms_convo_cat_name_extract" class="disabled">SMS Conversation Categorization and Name Extraction</a></li>
             </ul>
         </div>
 
diff --git a/requirements.txt b/requirements.txt
index 496c04e..94b8308 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+celery==5.0.5
 coverage==5.4
 Flask==1.1.2
 matplotlib==3.3.3

From 95252b6e354abcc9e9ce8dc19a54f9ad8b4a5f67 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 12 Mar 2021 13:34:45 -0700
Subject: [PATCH 21/52] template and settings updates

---
 parser/templates/upload_form.html | 2 +-
 settings.py.example               | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 0749b61..edd7df2 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -80,7 +80,7 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
                 </div>
             </form>
 
-            <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+            <p>If successful, output will be a CSV with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
 
         </div>
 
diff --git a/settings.py.example b/settings.py.example
index 9a77ab3..2a179d4 100644
--- a/settings.py.example
+++ b/settings.py.example
@@ -1,3 +1,2 @@
-ALLOWED_EXTENSIONS = {'csv'}
 SECRET_KEY = '' # keep this secret in production!
 MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16 MB

From dc3d007dbac77a89c91dbebb132841286911ebde Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 12 Mar 2021 13:34:45 -0700
Subject: [PATCH 22/52] template and settings updates

---
 parser/templates/upload_form.html | 2 +-
 settings.py.example               | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 0749b61..edd7df2 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -80,7 +80,7 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
                 </div>
             </form>
 
-            <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+            <p>If successful, output will be a CSV with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
 
         </div>
 
diff --git a/settings.py.example b/settings.py.example
index 9a77ab3..2a179d4 100644
--- a/settings.py.example
+++ b/settings.py.example
@@ -1,3 +1,2 @@
-ALLOWED_EXTENSIONS = {'csv'}
 SECRET_KEY = '' # keep this secret in production!
 MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16 MB

From aca08e0fbb91071e03f3bc933fc8d962e32d2497 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 25 Mar 2021 12:45:16 -0600
Subject: [PATCH 23/52] whitespace and some dirname changes bc case matters in
 linux

---
 .gitignore                                     |  3 ++-
 .../NLP/SMS_Annotation/Input_Data/readme.txt   |  1 +
 README_local.md                                | 18 +++++++++---------
 parser/__init__.py                             |  4 ++--
 4 files changed, 14 insertions(+), 12 deletions(-)
 create mode 100644 Projects/NLP/SMS_Annotation/Input_Data/readme.txt

diff --git a/.gitignore b/.gitignore
index 36dba15..faf19dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 .DS_Store
 *.csv
+*.xlsx
 settings.py
 
 instance/
@@ -13,4 +14,4 @@ htmlcov/
 
 dist/
 build/
-*.egg-info/
+*.egg-info/
\ No newline at end of file
diff --git a/Projects/NLP/SMS_Annotation/Input_Data/readme.txt b/Projects/NLP/SMS_Annotation/Input_Data/readme.txt
new file mode 100644
index 0000000..770703e
--- /dev/null
+++ b/Projects/NLP/SMS_Annotation/Input_Data/readme.txt
@@ -0,0 +1 @@
+Uploaded data will show up in this dir!
\ No newline at end of file
diff --git a/README_local.md b/README_local.md
index 27fac23..3821812 100644
--- a/README_local.md
+++ b/README_local.md
@@ -11,7 +11,7 @@ This document describes how to use 5 versions of name extraction scripts for vot
   - Levenshtein `pip install python-Levenshtein`
   - NLTK `pip install nltk`
 
-- Alternatively, if you are in an environment where you can't / don't want to install Anaconda, create and activate your Python 3 virtualenv (see [pipenv and virtualenv](https://docs.python-guide.org/dev/virtualenvs/)) and run `pip install -r requirements.txt`
+- Alternatively, if you are in an environment where you can't / don't want to install Anaconda, install Python 3.6.9+. Create and activate your Python 3 virtual environment (see [pipenv and virtualenv](https://docs.python-guide.org/dev/virtualenvs/)) and run `pip install -r requirements.txt`
 
 - You'll also need to run `spacy download en` once.
 
@@ -25,7 +25,7 @@ All output data (after running a script) will be found in `Projects/NLP/SMS_Anno
 **Use Case:** I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.  
   
 **Inputs:**
-Add a csv added to the Input_Data folder. This csv should be raw individual SMS messages, not grouped by conversation.  
+Add a csv added to the Input_Data folder. This csv should be raw individual SMS messages, not grouped by conversation.
 
 **Instructions:**
 Open the script aggregate_text_messages.R in RStudioo and follow the instructions to aggregate messages into a single row per conversation
@@ -90,11 +90,10 @@ In this directory, run `python3 Code/name_cleaning.py -i [input_filename]`
 
 **Outputs:**
 A file in `Output_Data` named `labeled_names_cleaned_no_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file 
-  
-  
+
 ## Text Banker Log Cleaning (utilizing text message conversation)
-**Use Case:** I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.  
-  
+**Use Case:** I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.
+
 **Inputs:**
 Add a csv to the Input_Data folder. 
 This csv file must be of the same format as the output of the aggregation in step 1.
@@ -105,8 +104,7 @@ In this directory, run `python3 Code/name_cleaning_with_responses.py -d [input_f
 
 **Outputs:**
 A File named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file
-  
-  
+
 ## VAN Export Cleaning
 **Use Case:** I have a VAN Export and I need to extract any tripling target names from the note text.
 
@@ -117,7 +115,7 @@ Add a csv to the Input_Data folder. This csv file must contain the following col
 - *NoteText* free text possibly including names of tripling targets
 
 **Instructions:**
-In this directory, run `python3 Code/van_export_cleaning.py -d [input_filename]`  
+In this directory, run `python3 Code/van_export_cleaning.py -d [input_filename]`
 
 **Outputs:**
 This script will output two files:  
@@ -136,6 +134,8 @@ app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for up
 
 Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
 
+If your (Centos or Redhat Linux) machine's Python3.7 doesn't come with sqlite3, you may have to install sqlite3 with `pip install pysqlite3`, then do `sudo yum install sqlite-devel`, then navigate to the source folder for Python 3.7 (`/usr/src` or something like that) and do `./configure` and then `make altinstall` to recompile Python 3.7 with sqlite3.
+
 To run an instance of the frontend locally, from the project root directory initialize the db:
 `
 export FLASK_APP=parser
diff --git a/parser/__init__.py b/parser/__init__.py
index 8cc4947..4d2ad07 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -12,8 +12,8 @@ def create_app(test_config=None):
         SECRET_KEY=settings.SECRET_KEY,
         DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
         MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
-        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Input_data'),
-        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_data'),
+        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Input_Data'),
+        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_Data'),
         SCRIPTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Code')
         # CELERY_BROKER_URL=,
         # CELERY_RESULT_BACKEND=

From 2c7f6a9c7d67c0cb73379a9b40d932a6da6a8ab3 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 25 Mar 2021 12:45:16 -0600
Subject: [PATCH 24/52] whitespace and some dirname changes bc case matters in
 linux

---
 .gitignore                                     |  3 ++-
 .../NLP/SMS_Annotation/Input_Data/readme.txt   |  1 +
 README_local.md                                | 18 +++++++++---------
 parser/__init__.py                             |  4 ++--
 4 files changed, 14 insertions(+), 12 deletions(-)
 create mode 100644 Projects/NLP/SMS_Annotation/Input_Data/readme.txt

diff --git a/.gitignore b/.gitignore
index 36dba15..faf19dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 .DS_Store
 *.csv
+*.xlsx
 settings.py
 
 instance/
@@ -13,4 +14,4 @@ htmlcov/
 
 dist/
 build/
-*.egg-info/
+*.egg-info/
\ No newline at end of file
diff --git a/Projects/NLP/SMS_Annotation/Input_Data/readme.txt b/Projects/NLP/SMS_Annotation/Input_Data/readme.txt
new file mode 100644
index 0000000..770703e
--- /dev/null
+++ b/Projects/NLP/SMS_Annotation/Input_Data/readme.txt
@@ -0,0 +1 @@
+Uploaded data will show up in this dir!
\ No newline at end of file
diff --git a/README_local.md b/README_local.md
index 27fac23..3821812 100644
--- a/README_local.md
+++ b/README_local.md
@@ -11,7 +11,7 @@ This document describes how to use 5 versions of name extraction scripts for vot
   - Levenshtein `pip install python-Levenshtein`
   - NLTK `pip install nltk`
 
-- Alternatively, if you are in an environment where you can't / don't want to install Anaconda, create and activate your Python 3 virtualenv (see [pipenv and virtualenv](https://docs.python-guide.org/dev/virtualenvs/)) and run `pip install -r requirements.txt`
+- Alternatively, if you are in an environment where you can't / don't want to install Anaconda, install Python 3.6.9+. Create and activate your Python 3 virtual environment (see [pipenv and virtualenv](https://docs.python-guide.org/dev/virtualenvs/)) and run `pip install -r requirements.txt`
 
 - You'll also need to run `spacy download en` once.
 
@@ -25,7 +25,7 @@ All output data (after running a script) will be found in `Projects/NLP/SMS_Anno
 **Use Case:** I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.  
   
 **Inputs:**
-Add a csv added to the Input_Data folder. This csv should be raw individual SMS messages, not grouped by conversation.  
+Add a csv added to the Input_Data folder. This csv should be raw individual SMS messages, not grouped by conversation.
 
 **Instructions:**
 Open the script aggregate_text_messages.R in RStudioo and follow the instructions to aggregate messages into a single row per conversation
@@ -90,11 +90,10 @@ In this directory, run `python3 Code/name_cleaning.py -i [input_filename]`
 
 **Outputs:**
 A file in `Output_Data` named `labeled_names_cleaned_no_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file 
-  
-  
+
 ## Text Banker Log Cleaning (utilizing text message conversation)
-**Use Case:** I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.  
-  
+**Use Case:** I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.
+
 **Inputs:**
 Add a csv to the Input_Data folder. 
 This csv file must be of the same format as the output of the aggregation in step 1.
@@ -105,8 +104,7 @@ In this directory, run `python3 Code/name_cleaning_with_responses.py -d [input_f
 
 **Outputs:**
 A File named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file
-  
-  
+
 ## VAN Export Cleaning
 **Use Case:** I have a VAN Export and I need to extract any tripling target names from the note text.
 
@@ -117,7 +115,7 @@ Add a csv to the Input_Data folder. This csv file must contain the following col
 - *NoteText* free text possibly including names of tripling targets
 
 **Instructions:**
-In this directory, run `python3 Code/van_export_cleaning.py -d [input_filename]`  
+In this directory, run `python3 Code/van_export_cleaning.py -d [input_filename]`
 
 **Outputs:**
 This script will output two files:  
@@ -136,6 +134,8 @@ app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for up
 
 Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
 
+If your (Centos or Redhat Linux) machine's Python3.7 doesn't come with sqlite3, you may have to install sqlite3 with `pip install pysqlite3`, then do `sudo yum install sqlite-devel`, then navigate to the source folder for Python 3.7 (`/usr/src` or something like that) and do `./configure` and then `make altinstall` to recompile Python 3.7 with sqlite3.
+
 To run an instance of the frontend locally, from the project root directory initialize the db:
 `
 export FLASK_APP=parser
diff --git a/parser/__init__.py b/parser/__init__.py
index 8cc4947..4d2ad07 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -12,8 +12,8 @@ def create_app(test_config=None):
         SECRET_KEY=settings.SECRET_KEY,
         DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
         MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
-        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Input_data'),
-        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_data'),
+        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Input_Data'),
+        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_Data'),
         SCRIPTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Code')
         # CELERY_BROKER_URL=,
         # CELERY_RESULT_BACKEND=

From 4e3eacd4796f728d7a9f03c27e6c90a594c68142 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Tue, 6 Apr 2021 16:21:03 -0600
Subject: [PATCH 25/52] refactor file naming

---
 parser/main.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/parser/main.py b/parser/main.py
index 7e341d9..0bd74a7 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -81,9 +81,7 @@ def process_job():
     scripts_folder = current_app.config['SCRIPTS_FOLDER']
     scripts_home_dir = os.path.dirname(scripts_folder)
     print('shd', scripts_home_dir)
-    output_file = '{}-{}.csv'.format(''.join(random.SystemRandom().choice(
-        string.ascii_lowercase + string.digits) for _ in range(17)), 
-        datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
+    output_file = unique_filename()
     if job_type == 'tblc_file':
         cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
@@ -112,8 +110,8 @@ def process_job():
     done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
     update = db.query_db(done_query, (status, result_file, job_id), write=True)
     if status == 'success':
-        return output_file
-    return False
+        return output_file, True
+    return err_log, False
 
 
 def email_results():
@@ -147,12 +145,12 @@ def index():
             if not (file and allowed_file(file)):
                 flash('Invalid file type. Select a CSV to upload', 'error')
                 return redirect(request.url)
-            # if not check_headers(file, upload_type):
-            #     flash('Invalid CSV headers. Check CSV for required columns.', 'error')
-            #     return redirect(request.url)
+            if not check_headers(file, upload_type):
+                flash('Invalid CSV headers. Check CSV for required columns.', 'error')
+                return redirect(request.url)
 
             # if it passes all the checks, save file and queue for processing
-            filename = secure_filename(file.filename)
+            filename = unique_filename()
             file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
             file.save(file_path)
             email_name = '{}_email'.format(upload_type.split('_')[0])
@@ -169,12 +167,13 @@ def index():
             # queue_msg = ('Processing file {} as {}'.format(
             #                 filename, UPLOAD_TYPES[upload_type]['name']))
             # flash(queue_msg, 'info')
-            success = process_job()
+            process, success = process_job()
             if success:
                 outcome_msg = Markup(
-                    '<a href="/results/{}">Download results</a>'.format(success))
+                    '<a href="/results/{}">Download results</a>'.format(process))
             else:
-                outcome_msg = 'Error processing file'
+                outcome_msg = 'Error processing file {}'.format(
+                    process) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
             flash(outcome_msg, 'info')
             return redirect(request.url)
     # GET

From 48b63e9dc4907a391572b7a798b63bb3bbcb8b5a Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Tue, 6 Apr 2021 16:21:03 -0600
Subject: [PATCH 26/52] refactor file naming

---
 parser/main.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/parser/main.py b/parser/main.py
index 7e341d9..0bd74a7 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -81,9 +81,7 @@ def process_job():
     scripts_folder = current_app.config['SCRIPTS_FOLDER']
     scripts_home_dir = os.path.dirname(scripts_folder)
     print('shd', scripts_home_dir)
-    output_file = '{}-{}.csv'.format(''.join(random.SystemRandom().choice(
-        string.ascii_lowercase + string.digits) for _ in range(17)), 
-        datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
+    output_file = unique_filename()
     if job_type == 'tblc_file':
         cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
@@ -112,8 +110,8 @@ def process_job():
     done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
     update = db.query_db(done_query, (status, result_file, job_id), write=True)
     if status == 'success':
-        return output_file
-    return False
+        return output_file, True
+    return err_log, False
 
 
 def email_results():
@@ -147,12 +145,12 @@ def index():
             if not (file and allowed_file(file)):
                 flash('Invalid file type. Select a CSV to upload', 'error')
                 return redirect(request.url)
-            # if not check_headers(file, upload_type):
-            #     flash('Invalid CSV headers. Check CSV for required columns.', 'error')
-            #     return redirect(request.url)
+            if not check_headers(file, upload_type):
+                flash('Invalid CSV headers. Check CSV for required columns.', 'error')
+                return redirect(request.url)
 
             # if it passes all the checks, save file and queue for processing
-            filename = secure_filename(file.filename)
+            filename = unique_filename()
             file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
             file.save(file_path)
             email_name = '{}_email'.format(upload_type.split('_')[0])
@@ -169,12 +167,13 @@ def index():
             # queue_msg = ('Processing file {} as {}'.format(
             #                 filename, UPLOAD_TYPES[upload_type]['name']))
             # flash(queue_msg, 'info')
-            success = process_job()
+            process, success = process_job()
             if success:
                 outcome_msg = Markup(
-                    '<a href="/results/{}">Download results</a>'.format(success))
+                    '<a href="/results/{}">Download results</a>'.format(process))
             else:
-                outcome_msg = 'Error processing file'
+                outcome_msg = 'Error processing file {}'.format(
+                    process) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
             flash(outcome_msg, 'info')
             return redirect(request.url)
     # GET

From cc9c93a2701f922cbc3d180ca7b21b7f51bb4cc3 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 9 Apr 2021 10:45:07 -0600
Subject: [PATCH 27/52] fix bugs with unique filename and file stream pointer

---
 parser/main.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/parser/main.py b/parser/main.py
index 0bd74a7..0d142fe 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -41,10 +41,17 @@ def allowed_file(file):
         and file.filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
         and file.content_type == 'text/csv')
 
+def unique_filename():
+    return secure_filename(
+        '{}-{}.csv'.format(
+            ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(17)),
+            datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+        )
+    )
 
 def check_headers(file, upload_type):
     headers = file.readline().decode('utf-8').rstrip().split(',')
-    for header in UPLOAD_TYPES['required_headers']:
+    for header in UPLOAD_TYPES[upload_type]['required_headers']:
         if header not in headers:
             return False
     return True
@@ -152,6 +159,7 @@ def index():
             # if it passes all the checks, save file and queue for processing
             filename = unique_filename()
             file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
+            file.seek(0) # return pointer to beginning of file
             file.save(file_path)
             email_name = '{}_email'.format(upload_type.split('_')[0])
             email = request.form[email_name]

From f1d5c6791d79ed89f3f65084066e430a0d451fa8 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 9 Apr 2021 10:45:07 -0600
Subject: [PATCH 28/52] fix bugs with unique filename and file stream pointer

---
 parser/main.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/parser/main.py b/parser/main.py
index 0bd74a7..0d142fe 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -41,10 +41,17 @@ def allowed_file(file):
         and file.filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
         and file.content_type == 'text/csv')
 
+def unique_filename():
+    return secure_filename(
+        '{}-{}.csv'.format(
+            ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(17)),
+            datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+        )
+    )
 
 def check_headers(file, upload_type):
     headers = file.readline().decode('utf-8').rstrip().split(',')
-    for header in UPLOAD_TYPES['required_headers']:
+    for header in UPLOAD_TYPES[upload_type]['required_headers']:
         if header not in headers:
             return False
     return True
@@ -152,6 +159,7 @@ def index():
             # if it passes all the checks, save file and queue for processing
             filename = unique_filename()
             file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
+            file.seek(0) # return pointer to beginning of file
             file.save(file_path)
             email_name = '{}_email'.format(upload_type.split('_')[0])
             email = request.form[email_name]

From 7c7d46e7f0022652595535ea3f155d944a9ebd26 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Tue, 13 Apr 2021 13:17:12 -0600
Subject: [PATCH 29/52] enable VAN export cleaning and support two output files

---
 README_local.md                   |   2 +-
 parser/main.py                    |  55 +++++++++----
 parser/templates/upload_form.html | 128 ++++++++++++++++++++----------
 3 files changed, 127 insertions(+), 58 deletions(-)

diff --git a/README_local.md b/README_local.md
index 3821812..2f4e6f2 100644
--- a/README_local.md
+++ b/README_local.md
@@ -110,7 +110,7 @@ A File named `labeled_names_cleaned_with_response.csv` with the cleaned names in
 
 **Inputs:**
 Add a csv to the Input_Data folder. This csv file must contain the following columns:
-- *VANID* a unique ID for this row
+- *voter_file_vanid* a unique ID for this row
 - *ContactName* the name of the tripler
 - *NoteText* free text possibly including names of tripling targets
 
diff --git a/parser/main.py b/parser/main.py
index 0d142fe..10c247a 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -32,6 +32,10 @@
     'sms_agg_file': {
         'name': 'SMS Aggregation',
         'required_headers': []
+    },
+    'vec_file': {
+        'name': 'VAN Export Cleaning',
+        'required_headers': ['voter_file_vanid','ContactName','NoteText']
     }
 }
 
@@ -83,21 +87,28 @@ def process_job():
     processing_query = 'UPDATE jobs SET status = ? WHERE id = ?;'
     update = db.query_db(processing_query, ('processing', job_id), write=True)
 
-    # It seems silly to run these via the command line but it's for ease of 
-    # maintenance by non-engineers.
+    # It seems silly to run these via the command line
+    # but we can easily change that later.
     scripts_folder = current_app.config['SCRIPTS_FOLDER']
     scripts_home_dir = os.path.dirname(scripts_folder)
     print('shd', scripts_home_dir)
     output_file = unique_filename()
+    second_output_file = None
+    if job_type == 'vec_file':
+        second_output_file = unique_filename()
+
     if job_type == 'tblc_file':
         cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
-    elif job_type == 'tblc_tmc_file':
-        cmd = 'python {}/name_cleaning_with_responses.py'.format(scripts_folder)
-        args = ['-d {}'.format(input_file)]
-    elif job_type == 'sccne_file':
-        cmd = 'python {}/annotate_conversations.py'.format(scripts_folder)
-        args = ['-d {}'.format(input_file)]
+    elif job_type == 'vec_file':
+        cmd = 'python {}/van_export_cleaning.py -i {} -f {} -o {} -m {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file, second_output_file)
+    # elif job_type == 'tblc_tmc_file':
+    #     cmd = 'python {}/name_cleaning_with_responses.py'.format(scripts_folder)
+    #     args = ['-d {}'.format(input_file)]
+    # elif job_type == 'sccne_file':
+    #     cmd = 'python {}/annotate_conversations.py'.format(scripts_folder)
+    #     args = ['-d {}'.format(input_file)]
     # elif job_type == 'sms_agg_file':
         # this script is to be ported to Python
     else:
@@ -112,13 +123,17 @@ def process_job():
         print(err_log)
     else:
         print("SUCCESS")
-    result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], 
-        output_file) if status == 'success' else None
+    result_file = None
+    if status == 'success':
+        result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], output_file)
+        if second_output_file:
+            result_file == '{0}/{1}|{0}/{2}'.format(
+                current_app.config['RESULTS_FOLDER'], output_file, second_output_file)
     done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
     update = db.query_db(done_query, (status, result_file, job_id), write=True)
     if status == 'success':
-        return output_file, True
-    return err_log, False
+        return True, output_file, second_output_file
+    return False, err_log, None
 
 
 def email_results():
@@ -145,7 +160,6 @@ def index():
             if upload_type not in UPLOAD_TYPES:
                 return redirect(request.url)
             file = request.files[upload_type]
-            # If selected file is empty / without filename
             if file.filename == '':
                 flash('No selected file', 'error')
                 return redirect(request.url)
@@ -153,7 +167,9 @@ def index():
                 flash('Invalid file type. Select a CSV to upload', 'error')
                 return redirect(request.url)
             if not check_headers(file, upload_type):
-                flash('Invalid CSV headers. Check CSV for required columns.', 'error')
+                msg = 'Invalid CSV headers. Check CSV for required columns {}.'.format(
+                    UPLOAD_TYPES[upload_type]['required_headers'])
+                flash(msg, 'error')
                 return redirect(request.url)
 
             # if it passes all the checks, save file and queue for processing
@@ -175,13 +191,18 @@ def index():
             # queue_msg = ('Processing file {} as {}'.format(
             #                 filename, UPLOAD_TYPES[upload_type]['name']))
             # flash(queue_msg, 'info')
-            process, success = process_job()
+            success, output, second_output = process_job()
             if success:
                 outcome_msg = Markup(
-                    '<a href="/results/{}">Download results</a>'.format(process))
+                    '<a href="/results/{}">Download results</a>'.format(output))
+                if second_output:
+                    outcome_msg = Markup(
+                        'Download <a href="/results/{}">result file 1</a>'
+                        ' and <a href="/results/{}">result file 2</a>').format(
+                            output, second_output)
             else:
                 outcome_msg = 'Error processing file {}'.format(
-                    process) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
+                    output) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
             flash(outcome_msg, 'info')
             return redirect(request.url)
     # GET
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index edd7df2..5134372 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -26,6 +26,7 @@
         blockquote, form { display: block;  margin-left: 0; padding: 1em; color: #444; background-color: rgba(255,255,255,.8); max-width: 90%; }
         div.upload-disabled { opacity: .3; }
         div.upload-disabled form {display: none;}
+        strong {font-size: 1.1em;}
     </style>
 
     <script type="text/javascript">
@@ -52,8 +53,9 @@ <h1>Vote tripling SMS transcript parsing</h1>
 
             <ul class="menu">
                 <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
                 <li><a href="#text_banker_log_cleaning_with_conversation" class="disabled">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
-                <li><a href="#van_export_cleaning" class="disabled">VAN export cleaning</a></li>
+                <li><a href="#sms_agg" class="disabled">SMS Aggregation</a></li>
                 <li><a href="#sms_convo_cat_name_extract" class="disabled">SMS Conversation Categorization and Name Extraction</a></li>
             </ul>
         </div>
@@ -66,7 +68,7 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
             <blockquote> I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.</blockquote>
 
             <p>First, enter your email address to receive a link to the output file.</p>
-            <p>Then, upload a csv containing a column 'names' containing the names logged by a text banker.</p>
+            <p>Then, upload a csv containing a column <strong>names</strong> containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
                 <div>
@@ -80,10 +82,57 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
                 </div>
             </form>
 
-            <p>If successful, output will be a CSV with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+            <p>If successful, output will be a CSV with the cleaned names in a column titled <strong>clean_names</strong>, along with any other columns in the initial file.</p>
 
         </div>
 
+        <div class="upload-enabled">
+            <hr />
+
+            <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
+
+            <blockquote> I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
+
+            <p>Upload a csv file containing the following columns:</p>
+            <ul>
+                <li><strong>voter_file_vanid:</strong> a unique ID for this row</li>
+                <li><strong>ContactName:</strong> the name of the tripler</li>
+                <li><strong>NoteText:</strong> free text possibly including names of tripling targets</li>
+            </ul>
+
+            <form method="post" enctype="multipart/form-data">
+                <div>
+                    <label for="tblc_email">Your email:</label>
+                    <input type="email" id="vec_email" name="vec_email" size="30">
+                </div>
+                <div>
+                    <label for="tblc_file">Your CSV:</label>
+                    <input type=file id="vec_file" name="vec_file" accept=".csv">
+                    <input type=submit value=Upload>
+                </div>
+            </form>
+
+            <p>This script will output two files:
+            <ul>
+                <li>A CSV file of triplers. For each tripler, we provide the following fields (each row represents one text message conversation):
+                    <ul>
+                        <li><strong>VANID:</strong> a unique identifier for the conversation</li>
+                        <li><strong>names_extract:</strong> the extracted names</li>
+                    </ul>
+                </li>
+                <li>A CSV file of conversations for manual review, with the following fields:
+                    <ul>
+                        <li><strong>VANID:</strong> a unique identifier for the conversation</li>
+                        <li><strong>ContactName:</strong> a unique identifier for the conversation</li>
+                        <li><strong>NoteText:</strong> free text possibly including names of tripling targets</li>
+                        <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)</li>
+                    </ul>
+                </li>
+            </ul>
+
+        </div>
+
+
         <div class="upload-disabled">
             <hr />
 
@@ -93,15 +142,48 @@ <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (ut
 
             <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
 
-            <p>Upload a csv file of the same format as the output of the aggregation in step 1.
-            This csv file must also contain column 'names' containing the names logged by a text banker.</p>
+            <p>Upload a CSV file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a>.
+            This CSV file must also contain column <strong>names</strong> containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
                 <input type=file id="tblc_tmc_file" name="tblc_tmc_file" accept=".csv">
                 <input type=submit value=Upload>
             </form>
 
-            <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+            <p>If successful, output will be a CSV with the cleaned names in a column titled <strong>clean_names</strong>, along with any other columns in the initial file.</p>
+
+        </div>
+
+        <div class="upload-disabled">
+            <hr />
+
+            <h2 id="sms_agg">SMS Aggregation</h2>
+            <blockquote>I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.</blockquote>
+
+            <p>Upload a csv file of raw individual SMS messages, not grouped by conversation.</p>
+
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="sms_agg_file" name="sms_agg_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
+
+            <a name="sms_agg_output"><p>If successful, the output will be a csv file with a single row representing each text message conversation, including the following fields:
+            <ul>
+                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                <li><strong>contact_phone:</strong> the phone number of the target</li>
+                <li><strong>totalMessages:</strong> the total number of messages exchanged</li>
+                <li><strong>tripleMessage:</strong> initial message sent from the text banker to the target</li>
+                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                <li><strong>tripleResponse:</strong> follow up message sent from the text banker to the target</li>
+                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                <li><strong>tripleFinal:</strong> final follow up sent by text banker</li>
+                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                <li><strong>noResponse:</strong> boolean for whether there was no response</li>
+                <li><strong>negResponse:</strong> boolean for generally negative or discouraging terms (sorry, no, etc.)</li>
+                <li><strong>posResponse:</strong> boolean for generally positive or encouraging terms</li>
+                <li><strong>affirmResponse:</strong> boolean for presence of a scripted affirmation by text banker</li>
+                <li><strong>finalAffirmResponse:</strong> boolean for presence of a scripted follow up affirmation by text banker</li>
+            </ul>
 
         </div>
 
@@ -144,40 +226,6 @@ <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Ext
             </ul>
         </div>
 
-        <div class="upload-disabled">
-            <hr />
-
-            <h2 id="sms_agg">SMS Aggregation</h2>
-            <blockquote>I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.</blockquote>
-
-            <p>Upload a csv file of raw individual SMS messages, not grouped by conversation.</p>
-
-            <form method="post" enctype="multipart/form-data">
-                <input type=file id="sms_agg_file" name="sms_agg_file" accept=".csv">
-                <input type=submit value=Upload>
-            </form>
-
-            <p>If successful, the output will be a csv file with a single row representing each text message conversation, including the following fields:
-            <ul>
-                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
-                <li><strong>contact_phone:</strong> the phone number of the target</li>
-                <li><strong>totalMessages:</strong> the total number of messages exchanged</li>
-                <li><strong>tripleMessage:</strong> initial message sent from the text banker to the target</li>
-                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
-                <li><strong>tripleResponse:</strong> follow up message sent from the text banker to the target</li>
-                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
-                <li><strong>tripleFinal:</strong> final follow up sent by text banker</li>
-                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
-                <li><strong>noResponse:</strong> boolean for whether there was no response</li>
-                <li><strong>negResponse:</strong> boolean for generally negative or discouraging terms (sorry, no, etc.)</li>
-                <li><strong>posResponse:</strong> boolean for generally positive or encouraging terms</li>
-                <li><strong>affirmResponse:</strong> boolean for presence of a scripted affirmation by text banker</li>
-                <li><strong>finalAffirmResponse:</strong> boolean for presence of a scripted follow up affirmation by text banker</li>
-            </ul>
-
-        </div>
-
-
 
     </main>
 

From 01a84cd5f73dd81b0ca73e03872f4af0d6c0f813 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Tue, 13 Apr 2021 13:17:12 -0600
Subject: [PATCH 30/52] enable VAN export cleaning and support two output files

---
 README_local.md                   |   2 +-
 parser/main.py                    |  55 +++++++++----
 parser/templates/upload_form.html | 128 ++++++++++++++++++++----------
 3 files changed, 127 insertions(+), 58 deletions(-)

diff --git a/README_local.md b/README_local.md
index 3821812..2f4e6f2 100644
--- a/README_local.md
+++ b/README_local.md
@@ -110,7 +110,7 @@ A File named `labeled_names_cleaned_with_response.csv` with the cleaned names in
 
 **Inputs:**
 Add a csv to the Input_Data folder. This csv file must contain the following columns:
-- *VANID* a unique ID for this row
+- *voter_file_vanid* a unique ID for this row
 - *ContactName* the name of the tripler
 - *NoteText* free text possibly including names of tripling targets
 
diff --git a/parser/main.py b/parser/main.py
index 0d142fe..10c247a 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -32,6 +32,10 @@
     'sms_agg_file': {
         'name': 'SMS Aggregation',
         'required_headers': []
+    },
+    'vec_file': {
+        'name': 'VAN Export Cleaning',
+        'required_headers': ['voter_file_vanid','ContactName','NoteText']
     }
 }
 
@@ -83,21 +87,28 @@ def process_job():
     processing_query = 'UPDATE jobs SET status = ? WHERE id = ?;'
     update = db.query_db(processing_query, ('processing', job_id), write=True)
 
-    # It seems silly to run these via the command line but it's for ease of 
-    # maintenance by non-engineers.
+    # It seems silly to run these via the command line
+    # but we can easily change that later.
     scripts_folder = current_app.config['SCRIPTS_FOLDER']
     scripts_home_dir = os.path.dirname(scripts_folder)
     print('shd', scripts_home_dir)
     output_file = unique_filename()
+    second_output_file = None
+    if job_type == 'vec_file':
+        second_output_file = unique_filename()
+
     if job_type == 'tblc_file':
         cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
-    elif job_type == 'tblc_tmc_file':
-        cmd = 'python {}/name_cleaning_with_responses.py'.format(scripts_folder)
-        args = ['-d {}'.format(input_file)]
-    elif job_type == 'sccne_file':
-        cmd = 'python {}/annotate_conversations.py'.format(scripts_folder)
-        args = ['-d {}'.format(input_file)]
+    elif job_type == 'vec_file':
+        cmd = 'python {}/van_export_cleaning.py -i {} -f {} -o {} -m {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file, second_output_file)
+    # elif job_type == 'tblc_tmc_file':
+    #     cmd = 'python {}/name_cleaning_with_responses.py'.format(scripts_folder)
+    #     args = ['-d {}'.format(input_file)]
+    # elif job_type == 'sccne_file':
+    #     cmd = 'python {}/annotate_conversations.py'.format(scripts_folder)
+    #     args = ['-d {}'.format(input_file)]
     # elif job_type == 'sms_agg_file':
         # this script is to be ported to Python
     else:
@@ -112,13 +123,17 @@ def process_job():
         print(err_log)
     else:
         print("SUCCESS")
-    result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], 
-        output_file) if status == 'success' else None
+    result_file = None
+    if status == 'success':
+        result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], output_file)
+        if second_output_file:
+            result_file == '{0}/{1}|{0}/{2}'.format(
+                current_app.config['RESULTS_FOLDER'], output_file, second_output_file)
     done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
     update = db.query_db(done_query, (status, result_file, job_id), write=True)
     if status == 'success':
-        return output_file, True
-    return err_log, False
+        return True, output_file, second_output_file
+    return False, err_log, None
 
 
 def email_results():
@@ -145,7 +160,6 @@ def index():
             if upload_type not in UPLOAD_TYPES:
                 return redirect(request.url)
             file = request.files[upload_type]
-            # If selected file is empty / without filename
             if file.filename == '':
                 flash('No selected file', 'error')
                 return redirect(request.url)
@@ -153,7 +167,9 @@ def index():
                 flash('Invalid file type. Select a CSV to upload', 'error')
                 return redirect(request.url)
             if not check_headers(file, upload_type):
-                flash('Invalid CSV headers. Check CSV for required columns.', 'error')
+                msg = 'Invalid CSV headers. Check CSV for required columns {}.'.format(
+                    UPLOAD_TYPES[upload_type]['required_headers'])
+                flash(msg, 'error')
                 return redirect(request.url)
 
             # if it passes all the checks, save file and queue for processing
@@ -175,13 +191,18 @@ def index():
             # queue_msg = ('Processing file {} as {}'.format(
             #                 filename, UPLOAD_TYPES[upload_type]['name']))
             # flash(queue_msg, 'info')
-            process, success = process_job()
+            success, output, second_output = process_job()
             if success:
                 outcome_msg = Markup(
-                    '<a href="/results/{}">Download results</a>'.format(process))
+                    '<a href="/results/{}">Download results</a>'.format(output))
+                if second_output:
+                    outcome_msg = Markup(
+                        'Download <a href="/results/{}">result file 1</a>'
+                        ' and <a href="/results/{}">result file 2</a>').format(
+                            output, second_output)
             else:
                 outcome_msg = 'Error processing file {}'.format(
-                    process) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
+                    output) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
             flash(outcome_msg, 'info')
             return redirect(request.url)
     # GET
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index edd7df2..5134372 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -26,6 +26,7 @@
         blockquote, form { display: block;  margin-left: 0; padding: 1em; color: #444; background-color: rgba(255,255,255,.8); max-width: 90%; }
         div.upload-disabled { opacity: .3; }
         div.upload-disabled form {display: none;}
+        strong {font-size: 1.1em;}
     </style>
 
     <script type="text/javascript">
@@ -52,8 +53,9 @@ <h1>Vote tripling SMS transcript parsing</h1>
 
             <ul class="menu">
                 <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+                <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
                 <li><a href="#text_banker_log_cleaning_with_conversation" class="disabled">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
-                <li><a href="#van_export_cleaning" class="disabled">VAN export cleaning</a></li>
+                <li><a href="#sms_agg" class="disabled">SMS Aggregation</a></li>
                 <li><a href="#sms_convo_cat_name_extract" class="disabled">SMS Conversation Categorization and Name Extraction</a></li>
             </ul>
         </div>
@@ -66,7 +68,7 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
             <blockquote> I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.</blockquote>
 
             <p>First, enter your email address to receive a link to the output file.</p>
-            <p>Then, upload a csv containing a column 'names' containing the names logged by a text banker.</p>
+            <p>Then, upload a csv containing a column <strong>names</strong> containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
                 <div>
@@ -80,10 +82,57 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
                 </div>
             </form>
 
-            <p>If successful, output will be a CSV with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+            <p>If successful, output will be a CSV with the cleaned names in a column titled <strong>clean_names</strong>, along with any other columns in the initial file.</p>
 
         </div>
 
+        <div class="upload-enabled">
+            <hr />
+
+            <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
+
+            <blockquote> I have a VAN Export and I need to extract any tripling target names from the note text.</blockquote>
+
+            <p>Upload a csv file containing the following columns:</p>
+            <ul>
+                <li><strong>voter_file_vanid:</strong> a unique ID for this row</li>
+                <li><strong>ContactName:</strong> the name of the tripler</li>
+                <li><strong>NoteText:</strong> free text possibly including names of tripling targets</li>
+            </ul>
+
+            <form method="post" enctype="multipart/form-data">
+                <div>
+                    <label for="tblc_email">Your email:</label>
+                    <input type="email" id="vec_email" name="vec_email" size="30">
+                </div>
+                <div>
+                    <label for="tblc_file">Your CSV:</label>
+                    <input type=file id="vec_file" name="vec_file" accept=".csv">
+                    <input type=submit value=Upload>
+                </div>
+            </form>
+
+            <p>This script will output two files:
+            <ul>
+                <li>A CSV file of triplers. For each tripler, we provide the following fields (each row represents one text message conversation):
+                    <ul>
+                        <li><strong>VANID:</strong> a unique identifier for the conversation</li>
+                        <li><strong>names_extract:</strong> the extracted names</li>
+                    </ul>
+                </li>
+                <li>A CSV file of conversations for manual review, with the following fields:
+                    <ul>
+                        <li><strong>VANID:</strong> a unique identifier for the conversation</li>
+                        <li><strong>ContactName:</strong> a unique identifier for the conversation</li>
+                        <li><strong>NoteText:</strong> free text possibly including names of tripling targets</li>
+                        <li><strong>names_extract:</strong> a guess for the extracted names (to be reviewed)</li>
+                    </ul>
+                </li>
+            </ul>
+
+        </div>
+
+
         <div class="upload-disabled">
             <hr />
 
@@ -93,15 +142,48 @@ <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (ut
 
             <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
 
-            <p>Upload a csv file of the same format as the output of the aggregation in step 1.
-            This csv file must also contain column 'names' containing the names logged by a text banker.</p>
+            <p>Upload a CSV file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a>.
+            This CSV file must also contain column <strong>names</strong> containing the names logged by a text banker.</p>
 
             <form method="post" enctype="multipart/form-data">
                 <input type=file id="tblc_tmc_file" name="tblc_tmc_file" accept=".csv">
                 <input type=submit value=Upload>
             </form>
 
-            <p>If successful, output will be a file named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titled "clean_names", along with any other columns in the initial file.</p>
+            <p>If successful, output will be a CSV with the cleaned names in a column titled <strong>clean_names</strong>, along with any other columns in the initial file.</p>
+
+        </div>
+
+        <div class="upload-disabled">
+            <hr />
+
+            <h2 id="sms_agg">SMS Aggregation</h2>
+            <blockquote>I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.</blockquote>
+
+            <p>Upload a csv file of raw individual SMS messages, not grouped by conversation.</p>
+
+            <form method="post" enctype="multipart/form-data">
+                <input type=file id="sms_agg_file" name="sms_agg_file" accept=".csv">
+                <input type=submit value=Upload>
+            </form>
+
+            <a name="sms_agg_output"><p>If successful, the output will be a csv file with a single row representing each text message conversation, including the following fields:
+            <ul>
+                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                <li><strong>contact_phone:</strong> the phone number of the target</li>
+                <li><strong>totalMessages:</strong> the total number of messages exchanged</li>
+                <li><strong>tripleMessage:</strong> initial message sent from the text banker to the target</li>
+                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                <li><strong>tripleResponse:</strong> follow up message sent from the text banker to the target</li>
+                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                <li><strong>tripleFinal:</strong> final follow up sent by text banker</li>
+                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                <li><strong>noResponse:</strong> boolean for whether there was no response</li>
+                <li><strong>negResponse:</strong> boolean for generally negative or discouraging terms (sorry, no, etc.)</li>
+                <li><strong>posResponse:</strong> boolean for generally positive or encouraging terms</li>
+                <li><strong>affirmResponse:</strong> boolean for presence of a scripted affirmation by text banker</li>
+                <li><strong>finalAffirmResponse:</strong> boolean for presence of a scripted follow up affirmation by text banker</li>
+            </ul>
 
         </div>
 
@@ -144,40 +226,6 @@ <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Ext
             </ul>
         </div>
 
-        <div class="upload-disabled">
-            <hr />
-
-            <h2 id="sms_agg">SMS Aggregation</h2>
-            <blockquote>I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.</blockquote>
-
-            <p>Upload a csv file of raw individual SMS messages, not grouped by conversation.</p>
-
-            <form method="post" enctype="multipart/form-data">
-                <input type=file id="sms_agg_file" name="sms_agg_file" accept=".csv">
-                <input type=submit value=Upload>
-            </form>
-
-            <p>If successful, the output will be a csv file with a single row representing each text message conversation, including the following fields:
-            <ul>
-                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
-                <li><strong>contact_phone:</strong> the phone number of the target</li>
-                <li><strong>totalMessages:</strong> the total number of messages exchanged</li>
-                <li><strong>tripleMessage:</strong> initial message sent from the text banker to the target</li>
-                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
-                <li><strong>tripleResponse:</strong> follow up message sent from the text banker to the target</li>
-                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
-                <li><strong>tripleFinal:</strong> final follow up sent by text banker</li>
-                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
-                <li><strong>noResponse:</strong> boolean for whether there was no response</li>
-                <li><strong>negResponse:</strong> boolean for generally negative or discouraging terms (sorry, no, etc.)</li>
-                <li><strong>posResponse:</strong> boolean for generally positive or encouraging terms</li>
-                <li><strong>affirmResponse:</strong> boolean for presence of a scripted affirmation by text banker</li>
-                <li><strong>finalAffirmResponse:</strong> boolean for presence of a scripted follow up affirmation by text banker</li>
-            </ul>
-
-        </div>
-
-
 
     </main>
 

From a5971d3170981b5b2b66f1b63c301a03b8242e7c Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 14 Apr 2021 12:00:56 -0600
Subject: [PATCH 31/52] added loader that you can't see

---
 parser/templates/upload_form.html | 63 ++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 21 deletions(-)

diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 5134372..8c4ea78 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -7,35 +7,43 @@
 
     <link href="https://fonts.googleapis.com/css?family=Droid+Serif|Roboto:500" rel="stylesheet">
     <style>
-        body { width: 100%; font-family: 'Droid Serif', serif; font-size: 16px; background-color: lightblue; }
-        h1,h2,h3,h4,h5,h6 { font-family: 'Roboto', sans-serif !important; }
+        body { width: 100%; font-family: 'Droid Serif', serif; font-size: 16px; background-color: lightblue; margin: 0; }
+        h1,h2,h3,h4,h5,h6,.loader { font-family: 'Roboto', sans-serif !important; }
         main { max-width: 700px; margin: auto; background-color: #eff; padding: 2em; }
         a { text-decoration: none; color: royalblue; }
         a:hover { text-decoration: underline; color: purple; }
         a:visited { color: rebeccapurple; }
-        form div { padding: .3em; margin: .3em .5em;}
-        ul.flash li {font-size: 1.3em;}
-        ul.flashes li.error {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
-        ul.flashes li.info {list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee;}
+        form div { padding: .3em; margin: .3em .5em; }
+        ul.flash li { font-size: 1.3em; }
+        ul.flashes li.error { list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee; }
+        ul.flashes li.info { list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee; }
         ul.flashes li.info a {text-decoration: underline; color: white;}
-        ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
-        ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
+        ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0; }
+        ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue; }
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
         ul.menu li a.disabled { opacity: 0.2; }
         input, button { font-size: .9em; height: 2em; }
         blockquote, form { display: block;  margin-left: 0; padding: 1em; color: #444; background-color: rgba(255,255,255,.8); max-width: 90%; }
-        div.upload-disabled { opacity: .3; }
-        div.upload-disabled form {display: none;}
-        strong {font-size: 1.1em;}
+        div.upload-disabled { opacity: .3;}
+        div.upload-disabled form { display: none; }
+        strong { font-size: 1.1em; }
+        #loader { position: fixed; height: 30px; top: 0px; background-color: gold; width: 100%; padding: 1em; text-align: center; font-size: 1.3em; color: black; transform: translate3d(0,0,0); opacity: .8; display: none;}
+        .spinner { border: 6px solid #f3f3f3; border-top: 6px solid green; border-radius: 50%; width: 20px; height: 20px; animation: spin 2s linear infinite; display: inline-block; }
+        @keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } }
+        .loader span { position: relative; top: -8px; left: 1em;}
+
     </style>
 
-    <script type="text/javascript">
-        // TODO: validation JS for forms
-    </script>
 </head>
 
 <body>
 
+    <div id="loader">
+        <div class="spinner">
+        </div>
+        <span>Processing ...</span>
+    </div>
+
     <main>
 
     {% with messages = get_flashed_messages(with_categories=true) %}
@@ -70,15 +78,15 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
             <p>First, enter your email address to receive a link to the output file.</p>
             <p>Then, upload a csv containing a column <strong>names</strong> containing the names logged by a text banker.</p>
 
-            <form method="post" enctype="multipart/form-data">
+            <form method="post" enctype="multipart/form-data" onsubmit="revealLoader();">
                 <div>
                     <label for="tblc_email">Your email:</label>
-                    <input type="email" id="tblc_email" name="tblc_email" size="30">
+                    <input type="email" id="tblc_email" name="tblc_email" size="30"/>
                 </div>
                 <div>
                     <label for="tblc_file">Your CSV:</label>
-                    <input type="file" id="tblc_file" name="tblc_file" accept=".csv">
-                    <input type=submit value=Upload>
+                    <input type="file" id="tblc_file" name="tblc_file" accept=".csv"/>
+                    <input type="submit" value="Upload" />
                 </div>
             </form>
 
@@ -103,12 +111,12 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
             <form method="post" enctype="multipart/form-data">
                 <div>
                     <label for="tblc_email">Your email:</label>
-                    <input type="email" id="vec_email" name="vec_email" size="30">
+                    <input type="email" id="vec_email" name="vec_email" size="30"/>
                 </div>
                 <div>
                     <label for="tblc_file">Your CSV:</label>
-                    <input type=file id="vec_file" name="vec_file" accept=".csv">
-                    <input type=submit value=Upload>
+                    <input type="file" id="vec_file" name="vec_file" accept=".csv"/>
+                    <input type="submit" value="Upload"/>
                 </div>
             </form>
 
@@ -229,6 +237,19 @@ <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Ext
 
     </main>
 
+    <script type="text/javascript">
+      // TODO: validation JS for forms
+      // TODO: reveal loader
+      function revealLoader() {
+        l = document.getElementById('loader');
+        if (l.style.display === "none") {
+          l.style.display = "block";
+        } else {
+          l.style.display = "none";
+        }
+      }
+    </script>
+
 </body>
 
 </html>
\ No newline at end of file

From 8ec3c608c4077a432247beccb43eb34be44ae534 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 14 Apr 2021 12:00:56 -0600
Subject: [PATCH 32/52] added loader that you can't see

---
 parser/templates/upload_form.html | 63 ++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 21 deletions(-)

diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 5134372..8c4ea78 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -7,35 +7,43 @@
 
     <link href="https://fonts.googleapis.com/css?family=Droid+Serif|Roboto:500" rel="stylesheet">
     <style>
-        body { width: 100%; font-family: 'Droid Serif', serif; font-size: 16px; background-color: lightblue; }
-        h1,h2,h3,h4,h5,h6 { font-family: 'Roboto', sans-serif !important; }
+        body { width: 100%; font-family: 'Droid Serif', serif; font-size: 16px; background-color: lightblue; margin: 0; }
+        h1,h2,h3,h4,h5,h6,.loader { font-family: 'Roboto', sans-serif !important; }
         main { max-width: 700px; margin: auto; background-color: #eff; padding: 2em; }
         a { text-decoration: none; color: royalblue; }
         a:hover { text-decoration: underline; color: purple; }
         a:visited { color: rebeccapurple; }
-        form div { padding: .3em; margin: .3em .5em;}
-        ul.flash li {font-size: 1.3em;}
-        ul.flashes li.error {list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee;}
-        ul.flashes li.info {list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee;}
+        form div { padding: .3em; margin: .3em .5em; }
+        ul.flash li { font-size: 1.3em; }
+        ul.flashes li.error { list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee; }
+        ul.flashes li.info { list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee; }
         ul.flashes li.info a {text-decoration: underline; color: white;}
-        ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0;}
-        ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue;}
+        ul.menu li { list-style-type: none; display: inline-block; margin: 0em .5em 2em 0; }
+        ul.menu li a { padding: .5em .9em; color: royalblue; background-color: transparent; font-size: 14px; font-family: 'Roboto', sans-serif; text-transform: uppercase; text-decoration: none; font-weight: bold; letter-spacing: .1em; border-radius: 1em; border: 2px solid royalblue; }
         ul.menu li a:hover { color: purple; border: 2px solid purple; }
         ul.menu li a.disabled { opacity: 0.2; }
         input, button { font-size: .9em; height: 2em; }
         blockquote, form { display: block;  margin-left: 0; padding: 1em; color: #444; background-color: rgba(255,255,255,.8); max-width: 90%; }
-        div.upload-disabled { opacity: .3; }
-        div.upload-disabled form {display: none;}
-        strong {font-size: 1.1em;}
+        div.upload-disabled { opacity: .3;}
+        div.upload-disabled form { display: none; }
+        strong { font-size: 1.1em; }
+        #loader { position: fixed; height: 30px; top: 0px; background-color: gold; width: 100%; padding: 1em; text-align: center; font-size: 1.3em; color: black; transform: translate3d(0,0,0); opacity: .8; display: none;}
+        .spinner { border: 6px solid #f3f3f3; border-top: 6px solid green; border-radius: 50%; width: 20px; height: 20px; animation: spin 2s linear infinite; display: inline-block; }
+        @keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } }
+        .loader span { position: relative; top: -8px; left: 1em;}
+
     </style>
 
-    <script type="text/javascript">
-        // TODO: validation JS for forms
-    </script>
 </head>
 
 <body>
 
+    <div id="loader">
+        <div class="spinner">
+        </div>
+        <span>Processing ...</span>
+    </div>
+
     <main>
 
     {% with messages = get_flashed_messages(with_categories=true) %}
@@ -70,15 +78,15 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
             <p>First, enter your email address to receive a link to the output file.</p>
             <p>Then, upload a csv containing a column <strong>names</strong> containing the names logged by a text banker.</p>
 
-            <form method="post" enctype="multipart/form-data">
+            <form method="post" enctype="multipart/form-data" onsubmit="revealLoader();">
                 <div>
                     <label for="tblc_email">Your email:</label>
-                    <input type="email" id="tblc_email" name="tblc_email" size="30">
+                    <input type="email" id="tblc_email" name="tblc_email" size="30"/>
                 </div>
                 <div>
                     <label for="tblc_file">Your CSV:</label>
-                    <input type="file" id="tblc_file" name="tblc_file" accept=".csv">
-                    <input type=submit value=Upload>
+                    <input type="file" id="tblc_file" name="tblc_file" accept=".csv"/>
+                    <input type="submit" value="Upload" />
                 </div>
             </form>
 
@@ -103,12 +111,12 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
             <form method="post" enctype="multipart/form-data">
                 <div>
                     <label for="tblc_email">Your email:</label>
-                    <input type="email" id="vec_email" name="vec_email" size="30">
+                    <input type="email" id="vec_email" name="vec_email" size="30"/>
                 </div>
                 <div>
                     <label for="tblc_file">Your CSV:</label>
-                    <input type=file id="vec_file" name="vec_file" accept=".csv">
-                    <input type=submit value=Upload>
+                    <input type="file" id="vec_file" name="vec_file" accept=".csv"/>
+                    <input type="submit" value="Upload"/>
                 </div>
             </form>
 
@@ -229,6 +237,19 @@ <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Ext
 
     </main>
 
+    <script type="text/javascript">
+      // TODO: validation JS for forms
+      // TODO: reveal loader
+      function revealLoader() {
+        l = document.getElementById('loader');
+        if (l.style.display === "none") {
+          l.style.display = "block";
+        } else {
+          l.style.display = "none";
+        }
+      }
+    </script>
+
 </body>
 
 </html>
\ No newline at end of file

From 1b59797a2df8146df5212751dd46b899a7473900 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 14 Apr 2021 13:34:49 -0600
Subject: [PATCH 33/52] in theory enabled two more upload types

---
 README_local.md                   | 20 +++----
 parser/main.py                    | 26 +++++----
 parser/templates/upload_form.html | 92 +++++++++++++++++++------------
 3 files changed, 82 insertions(+), 56 deletions(-)

diff --git a/README_local.md b/README_local.md
index 2f4e6f2..c60317e 100644
--- a/README_local.md
+++ b/README_local.md
@@ -49,16 +49,16 @@ A file (filename specified by you in the R script) with a single row representin
   
   
 ## SMS Conversation Categorization and Name Extraction
-**Use Case:** I have SMS conversations and I need to figure out which text recipiants volunteered to triple, which chose to opt out, what names they provided, and whether they moved.  
-  
+**Use Case:** I have SMS conversations and I need to figure out which text recipiants volunteered to triple, which chose to opt out, what names they provided, and whether they moved.
+
 **Inputs:**
-Add a csv to the Input_Data folder. This csv file must be of the same format as the output of the aggregation in step 1.   
+Add a CSV to the Input_Data folder. This csv file must be of the same format as the output of the aggregation in step 1.
 
 **Instructions:**
-In this directory, run `python3 Code/annotate_conversations.py -d [input_filename]`. 
+In this directory, run `python3 Code/annotate_conversations.py -i [input_filename]`.
 
 **Outputs:**
-This script will output two files:  
+This script will output two files:
 1. A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
 - *ConversationId* a unique identifier for the conversation
 - *contact_phone* the phone number of the target 
@@ -78,12 +78,12 @@ This script will output two files:
 - *wrong_number* guess for did we have the wrong number for this person (to be reviewed)
 - *names_extract* guess for what names (if any) were provided by this person as tripling targets (to be reviewed)
 
-  
+
 ## Text Banker Log Cleaning
-**Use Case:** I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.  
-  
+**Use Case:** I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.
+
 **Inputs:**
-Add a csv to the Input_Data folder. This csv file must contain column 'names' containing the names logged by a text banker  
+Add a csv to the Input_Data folder. This csv file must contain column 'names' containing the names logged by a text banker
 
 **Instructions:**
 In this directory, run `python3 Code/name_cleaning.py -i [input_filename]`
@@ -100,7 +100,7 @@ This csv file must be of the same format as the output of the aggregation in ste
 This csv file must also contain column 'names' containing the names logged by a text banker.
 
 **Instructions:**
-In this directory, run `python3 Code/name_cleaning_with_responses.py -d [input_filename]`
+In this directory, run `python3 Code/name_cleaning_with_responses.py -i [input_filename]`
 
 **Outputs:**
 A File named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file
diff --git a/parser/main.py b/parser/main.py
index 10c247a..27a16bb 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -21,15 +21,21 @@
         'name': 'Text Banker Log Cleaning',
         'required_headers': ['names']
     },
-    'tblc_tmc_file': {
+    'tblctmc_file': {
         'name': 'Text Banker Log Cleaning (utilizing text message conversation)',
-        'required_headers': ['names']
+        'required_headers': ['names', 'triplemessage', 'voterresponse',
+            'voterfinal', 'voterpost'
+        ]
     },
     'sccne_file': {
         'name': 'SMS Conversation Categorization and Name Extraction',
-        'required_headers': []
+        'required_headers': ['noresponse', 'negresponse', 'posresponse',
+            'affirmresponse', 'finalaffirmresponse', 'triplemessage',
+            'voterresponse','voterfinal', 'voterpost', 'conversationid',
+            'contact_phone'
+        ]
     },
-    'sms_agg_file': {
+    'smsagg_file': {
         'name': 'SMS Aggregation',
         'required_headers': []
     },
@@ -103,12 +109,12 @@ def process_job():
     elif job_type == 'vec_file':
         cmd = 'python {}/van_export_cleaning.py -i {} -f {} -o {} -m {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file, second_output_file)
-    # elif job_type == 'tblc_tmc_file':
-    #     cmd = 'python {}/name_cleaning_with_responses.py'.format(scripts_folder)
-    #     args = ['-d {}'.format(input_file)]
-    # elif job_type == 'sccne_file':
-    #     cmd = 'python {}/annotate_conversations.py'.format(scripts_folder)
-    #     args = ['-d {}'.format(input_file)]
+    elif job_type == 'tblc_tmc_file':
+        cmd = 'python {}/name_cleaning_with_responses.py -i {} -f {} -o {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file)
+    elif job_type == 'sccne_file':
+        cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file)
     # elif job_type == 'sms_agg_file':
         # this script is to be ported to Python
     else:
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 8c4ea78..e89e43a 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -61,10 +61,10 @@ <h1>Vote tripling SMS transcript parsing</h1>
 
             <ul class="menu">
                 <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+                <li><a href="#text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
                 <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
-                <li><a href="#text_banker_log_cleaning_with_conversation" class="disabled">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
                 <li><a href="#sms_agg" class="disabled">SMS Aggregation</a></li>
-                <li><a href="#sms_convo_cat_name_extract" class="disabled">SMS Conversation Categorization and Name Extraction</a></li>
+                <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
             </ul>
         </div>
 
@@ -78,7 +78,7 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
             <p>First, enter your email address to receive a link to the output file.</p>
             <p>Then, upload a csv containing a column <strong>names</strong> containing the names logged by a text banker.</p>
 
-            <form method="post" enctype="multipart/form-data" onsubmit="revealLoader();">
+            <form method="post" enctype="multipart/form-data">
                 <div>
                     <label for="tblc_email">Your email:</label>
                     <input type="email" id="tblc_email" name="tblc_email" size="30"/>
@@ -94,6 +94,34 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
 
         </div>
 
+        <div class="upload-enabled">
+            <hr />
+
+            <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</h2>
+
+            <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
+
+            <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
+
+            <p>Upload a CSV file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a>.This CSV file must also contain a column <strong>names</strong> containing the names logged by a text banker. You can join the text banker logs to the aggregation output using the conversation ID.</p>
+
+            <form method="post" enctype="multipart/form-data">
+                <div>
+                    <label for="tblctmc_email">Your email:</label>
+                    <input type="email" id="tblctmc_email" name="tblctmc_email" size="30"/>
+                </div>
+                <div>
+                    <label for="tblctmc_file">Your CSV:</label>
+                    <input type="file" id="tblctmc_file" name="tblctmc_file" accept=".csv" />
+                    <input type="submit" value="Upload" />
+                </div>
+
+            </form>
+
+            <p>If successful, output will be a CSV with the cleaned names in a column titled <strong>clean_names</strong>, along with any other columns in the initial file.</p>
+
+        </div>
+
         <div class="upload-enabled">
             <hr />
 
@@ -110,11 +138,11 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
 
             <form method="post" enctype="multipart/form-data">
                 <div>
-                    <label for="tblc_email">Your email:</label>
+                    <label for="vec_email">Your email:</label>
                     <input type="email" id="vec_email" name="vec_email" size="30"/>
                 </div>
                 <div>
-                    <label for="tblc_file">Your CSV:</label>
+                    <label for="vec_file">Your CSV:</label>
                     <input type="file" id="vec_file" name="vec_file" accept=".csv"/>
                     <input type="submit" value="Upload"/>
                 </div>
@@ -140,28 +168,6 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
 
         </div>
 
-
-        <div class="upload-disabled">
-            <hr />
-
-            <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</h2>
-
-            <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
-
-            <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
-
-            <p>Upload a CSV file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a>.
-            This CSV file must also contain column <strong>names</strong> containing the names logged by a text banker.</p>
-
-            <form method="post" enctype="multipart/form-data">
-                <input type=file id="tblc_tmc_file" name="tblc_tmc_file" accept=".csv">
-                <input type=submit value=Upload>
-            </form>
-
-            <p>If successful, output will be a CSV with the cleaned names in a column titled <strong>clean_names</strong>, along with any other columns in the initial file.</p>
-
-        </div>
-
         <div class="upload-disabled">
             <hr />
 
@@ -171,11 +177,18 @@ <h2 id="sms_agg">SMS Aggregation</h2>
             <p>Upload a csv file of raw individual SMS messages, not grouped by conversation.</p>
 
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="sms_agg_file" name="sms_agg_file" accept=".csv">
-                <input type=submit value=Upload>
+                <div>
+                    <label for="smsagg_email">Your email:</label>
+                    <input type="email" id="smsagg_email" name="smsagg_email" size="30"/>
+                </div>
+                <div>
+                    <label for="smsagg_file">Your CSV:</label>
+                    <input type="file" id="smsagg_file" name="smsagg_file" accept=".csv" />
+                    <input type="submit" value="Upload" />
+                </div>
             </form>
 
-            <a name="sms_agg_output"><p>If successful, the output will be a csv file with a single row representing each text message conversation, including the following fields:
+            <p id="sms_agg_output">If successful, the output will be a CSV with a single row representing each text message conversation, including the following fields:
             <ul>
                 <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
                 <li><strong>contact_phone:</strong> the phone number of the target</li>
@@ -195,21 +208,28 @@ <h2 id="sms_agg">SMS Aggregation</h2>
 
         </div>
 
-        <div class="upload-disabled">
+        <div class="upload-enabled">
             <hr />
             <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
             <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
 
-            <p>Upload a csv file of the same format as the output of the aggregation in step 1.</p>
+            <p>Upload a csv file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a></p>
             
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="sccne_file" name="sccne_file" accept=".csv">
-                <input type=submit value=Upload>
+                <div>
+                    <label for="sccne_email">Your email:</label>
+                    <input type="email" id="sccne_email" name="sccne_email" size="30"/>
+                </div>
+                <div>
+                    <label for="sccne_file">Your CSV:</label>
+                    <input type=file id="sccne_file" name="sccne_file" accept=".csv">
+                    <input type="submit" value="Upload"/>
+                </div>
             </form>
 
             <p>If successful, the output will include two files:
             <ul>
-                <li>A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+                <li>A file of triplers. For each tripler, we provide the following fields (each row represents one text message conversation):
                 <ul>
                     <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
                     <li><strong>contact_phone:</strong> the phone number of the target</li>
@@ -219,7 +239,7 @@ <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Ext
                     <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
                 </ul>
 
-                <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+                <li>A file of conversations for manual review, with the following fields:
                 <ul>
                     <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
                     <li><strong>contact_phone:</strong> the phone number of the target</li>

From 91a71c2e8aeb39d76f66628419bd0fdefcb6e79e Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 14 Apr 2021 13:34:49 -0600
Subject: [PATCH 34/52] in theory enabled two more upload types

---
 README_local.md                   | 20 +++----
 parser/main.py                    | 26 +++++----
 parser/templates/upload_form.html | 92 +++++++++++++++++++------------
 3 files changed, 82 insertions(+), 56 deletions(-)

diff --git a/README_local.md b/README_local.md
index 2f4e6f2..c60317e 100644
--- a/README_local.md
+++ b/README_local.md
@@ -49,16 +49,16 @@ A file (filename specified by you in the R script) with a single row representin
   
   
 ## SMS Conversation Categorization and Name Extraction
-**Use Case:** I have SMS conversations and I need to figure out which text recipiants volunteered to triple, which chose to opt out, what names they provided, and whether they moved.  
-  
+**Use Case:** I have SMS conversations and I need to figure out which text recipiants volunteered to triple, which chose to opt out, what names they provided, and whether they moved.
+
 **Inputs:**
-Add a csv to the Input_Data folder. This csv file must be of the same format as the output of the aggregation in step 1.   
+Add a CSV to the Input_Data folder. This csv file must be of the same format as the output of the aggregation in step 1.
 
 **Instructions:**
-In this directory, run `python3 Code/annotate_conversations.py -d [input_filename]`. 
+In this directory, run `python3 Code/annotate_conversations.py -i [input_filename]`.
 
 **Outputs:**
-This script will output two files:  
+This script will output two files:
 1. A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
 - *ConversationId* a unique identifier for the conversation
 - *contact_phone* the phone number of the target 
@@ -78,12 +78,12 @@ This script will output two files:
 - *wrong_number* guess for did we have the wrong number for this person (to be reviewed)
 - *names_extract* guess for what names (if any) were provided by this person as tripling targets (to be reviewed)
 
-  
+
 ## Text Banker Log Cleaning
-**Use Case:** I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.  
-  
+**Use Case:** I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.
+
 **Inputs:**
-Add a csv to the Input_Data folder. This csv file must contain column 'names' containing the names logged by a text banker  
+Add a csv to the Input_Data folder. This csv file must contain column 'names' containing the names logged by a text banker
 
 **Instructions:**
 In this directory, run `python3 Code/name_cleaning.py -i [input_filename]`
@@ -100,7 +100,7 @@ This csv file must be of the same format as the output of the aggregation in ste
 This csv file must also contain column 'names' containing the names logged by a text banker.
 
 **Instructions:**
-In this directory, run `python3 Code/name_cleaning_with_responses.py -d [input_filename]`
+In this directory, run `python3 Code/name_cleaning_with_responses.py -i [input_filename]`
 
 **Outputs:**
 A File named `labeled_names_cleaned_with_response.csv` with the cleaned names in a column titles "clean_names", along with any other columns in the initial file
diff --git a/parser/main.py b/parser/main.py
index 10c247a..27a16bb 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -21,15 +21,21 @@
         'name': 'Text Banker Log Cleaning',
         'required_headers': ['names']
     },
-    'tblc_tmc_file': {
+    'tblctmc_file': {
         'name': 'Text Banker Log Cleaning (utilizing text message conversation)',
-        'required_headers': ['names']
+        'required_headers': ['names', 'triplemessage', 'voterresponse',
+            'voterfinal', 'voterpost'
+        ]
     },
     'sccne_file': {
         'name': 'SMS Conversation Categorization and Name Extraction',
-        'required_headers': []
+        'required_headers': ['noresponse', 'negresponse', 'posresponse',
+            'affirmresponse', 'finalaffirmresponse', 'triplemessage',
+            'voterresponse','voterfinal', 'voterpost', 'conversationid',
+            'contact_phone'
+        ]
     },
-    'sms_agg_file': {
+    'smsagg_file': {
         'name': 'SMS Aggregation',
         'required_headers': []
     },
@@ -103,12 +109,12 @@ def process_job():
     elif job_type == 'vec_file':
         cmd = 'python {}/van_export_cleaning.py -i {} -f {} -o {} -m {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file, second_output_file)
-    # elif job_type == 'tblc_tmc_file':
-    #     cmd = 'python {}/name_cleaning_with_responses.py'.format(scripts_folder)
-    #     args = ['-d {}'.format(input_file)]
-    # elif job_type == 'sccne_file':
-    #     cmd = 'python {}/annotate_conversations.py'.format(scripts_folder)
-    #     args = ['-d {}'.format(input_file)]
+    elif job_type == 'tblc_tmc_file':
+        cmd = 'python {}/name_cleaning_with_responses.py -i {} -f {} -o {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file)
+    elif job_type == 'sccne_file':
+        cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file)
     # elif job_type == 'sms_agg_file':
         # this script is to be ported to Python
     else:
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 8c4ea78..e89e43a 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -61,10 +61,10 @@ <h1>Vote tripling SMS transcript parsing</h1>
 
             <ul class="menu">
                 <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
+                <li><a href="#text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
                 <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
-                <li><a href="#text_banker_log_cleaning_with_conversation" class="disabled">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
                 <li><a href="#sms_agg" class="disabled">SMS Aggregation</a></li>
-                <li><a href="#sms_convo_cat_name_extract" class="disabled">SMS Conversation Categorization and Name Extraction</a></li>
+                <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
             </ul>
         </div>
 
@@ -78,7 +78,7 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
             <p>First, enter your email address to receive a link to the output file.</p>
             <p>Then, upload a csv containing a column <strong>names</strong> containing the names logged by a text banker.</p>
 
-            <form method="post" enctype="multipart/form-data" onsubmit="revealLoader();">
+            <form method="post" enctype="multipart/form-data">
                 <div>
                     <label for="tblc_email">Your email:</label>
                     <input type="email" id="tblc_email" name="tblc_email" size="30"/>
@@ -94,6 +94,34 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
 
         </div>
 
+        <div class="upload-enabled">
+            <hr />
+
+            <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</h2>
+
+            <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
+
+            <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
+
+            <p>Upload a CSV file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a>.This CSV file must also contain a column <strong>names</strong> containing the names logged by a text banker. You can join the text banker logs to the aggregation output using the conversation ID.</p>
+
+            <form method="post" enctype="multipart/form-data">
+                <div>
+                    <label for="tblctmc_email">Your email:</label>
+                    <input type="email" id="tblctmc_email" name="tblctmc_email" size="30"/>
+                </div>
+                <div>
+                    <label for="tblctmc_file">Your CSV:</label>
+                    <input type="file" id="tblctmc_file" name="tblctmc_file" accept=".csv" />
+                    <input type="submit" value="Upload" />
+                </div>
+
+            </form>
+
+            <p>If successful, output will be a CSV with the cleaned names in a column titled <strong>clean_names</strong>, along with any other columns in the initial file.</p>
+
+        </div>
+
         <div class="upload-enabled">
             <hr />
 
@@ -110,11 +138,11 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
 
             <form method="post" enctype="multipart/form-data">
                 <div>
-                    <label for="tblc_email">Your email:</label>
+                    <label for="vec_email">Your email:</label>
                     <input type="email" id="vec_email" name="vec_email" size="30"/>
                 </div>
                 <div>
-                    <label for="tblc_file">Your CSV:</label>
+                    <label for="vec_file">Your CSV:</label>
                     <input type="file" id="vec_file" name="vec_file" accept=".csv"/>
                     <input type="submit" value="Upload"/>
                 </div>
@@ -140,28 +168,6 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
 
         </div>
 
-
-        <div class="upload-disabled">
-            <hr />
-
-            <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</h2>
-
-            <blockquote> I have text banker logs for names provided by vote triplers. I also have access to the initial text conversation. I need these logs cleaned up and standardized. </blockquote>
-
-            <p>We use a different script for these cases, because we can clean up the logs better and perform spell check by looking at the original messages.</p>
-
-            <p>Upload a CSV file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a>.
-            This CSV file must also contain column <strong>names</strong> containing the names logged by a text banker.</p>
-
-            <form method="post" enctype="multipart/form-data">
-                <input type=file id="tblc_tmc_file" name="tblc_tmc_file" accept=".csv">
-                <input type=submit value=Upload>
-            </form>
-
-            <p>If successful, output will be a CSV with the cleaned names in a column titled <strong>clean_names</strong>, along with any other columns in the initial file.</p>
-
-        </div>
-
         <div class="upload-disabled">
             <hr />
 
@@ -171,11 +177,18 @@ <h2 id="sms_agg">SMS Aggregation</h2>
             <p>Upload a csv file of raw individual SMS messages, not grouped by conversation.</p>
 
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="sms_agg_file" name="sms_agg_file" accept=".csv">
-                <input type=submit value=Upload>
+                <div>
+                    <label for="smsagg_email">Your email:</label>
+                    <input type="email" id="smsagg_email" name="smsagg_email" size="30"/>
+                </div>
+                <div>
+                    <label for="smsagg_file">Your CSV:</label>
+                    <input type="file" id="smsagg_file" name="smsagg_file" accept=".csv" />
+                    <input type="submit" value="Upload" />
+                </div>
             </form>
 
-            <a name="sms_agg_output"><p>If successful, the output will be a csv file with a single row representing each text message conversation, including the following fields:
+            <p id="sms_agg_output">If successful, the output will be a CSV with a single row representing each text message conversation, including the following fields:
             <ul>
                 <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
                 <li><strong>contact_phone:</strong> the phone number of the target</li>
@@ -195,21 +208,28 @@ <h2 id="sms_agg">SMS Aggregation</h2>
 
         </div>
 
-        <div class="upload-disabled">
+        <div class="upload-enabled">
             <hr />
             <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
             <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
 
-            <p>Upload a csv file of the same format as the output of the aggregation in step 1.</p>
+            <p>Upload a csv file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a></p>
             
             <form method="post" enctype="multipart/form-data">
-                <input type=file id="sccne_file" name="sccne_file" accept=".csv">
-                <input type=submit value=Upload>
+                <div>
+                    <label for="sccne_email">Your email:</label>
+                    <input type="email" id="sccne_email" name="sccne_email" size="30"/>
+                </div>
+                <div>
+                    <label for="sccne_file">Your CSV:</label>
+                    <input type=file id="sccne_file" name="sccne_file" accept=".csv">
+                    <input type="submit" value="Upload"/>
+                </div>
             </form>
 
             <p>If successful, the output will include two files:
             <ul>
-                <li>A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
+                <li>A file of triplers. For each tripler, we provide the following fields (each row represents one text message conversation):
                 <ul>
                     <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
                     <li><strong>contact_phone:</strong> the phone number of the target</li>
@@ -219,7 +239,7 @@ <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Ext
                     <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
                 </ul>
 
-                <li>A file of conversations for manual review called `van_manual_review.csv`, with the following fields:
+                <li>A file of conversations for manual review, with the following fields:
                 <ul>
                     <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
                     <li><strong>contact_phone:</strong> the phone number of the target</li>

From ede82a3cbe40d0bc26034e2fde23c1387a8daa24 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 22 Apr 2021 10:00:19 -0600
Subject: [PATCH 35/52] email sending works

---
 README_local.md         | 11 +++++++---
 parser/__init__.py      | 47 +++++++++++++++++++++++++++++++----------
 parser/celery_worker.py |  6 ++++++
 parser/main.py          |  2 +-
 requirements.txt        |  2 ++
 settings.py.example     | 10 +++++++++
 6 files changed, 63 insertions(+), 15 deletions(-)
 create mode 100644 parser/celery_worker.py

diff --git a/README_local.md b/README_local.md
index c60317e..90edbc7 100644
--- a/README_local.md
+++ b/README_local.md
@@ -130,11 +130,10 @@ This script will output two files:
 - *names_extract* a guess for the extracted names (to be reviewed)
 
 # Running the app frontend
-app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and running the scripts with them. It creates and manages a queue within a sqlite3 database (to be upgraded to Redis if performance needs call for it).
-
+app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and running the scripts with them. 
 Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
 
-If your (Centos or Redhat Linux) machine's Python3.7 doesn't come with sqlite3, you may have to install sqlite3 with `pip install pysqlite3`, then do `sudo yum install sqlite-devel`, then navigate to the source folder for Python 3.7 (`/usr/src` or something like that) and do `./configure` and then `make altinstall` to recompile Python 3.7 with sqlite3.
+You'll need to install Redis. On OSX, install homebrew and then `brew install redis`. You may also need to run `pip install "celery[redis]"`
 
 To run an instance of the frontend locally, from the project root directory initialize the db:
 `
@@ -147,6 +146,12 @@ That will create a file named `parser.sqlite3` (the application database) in the
 `
 and access the running application at [http://localhost:5000/](http://localhost:5000/)
 
+## Configuring and testing email sending
+
+Email config variables in the example config file assume you are using Gmail for testing. Two important notes:
+* Gmail probably isn't adequate for production scale; you can only send about 100 emails a day.
+* Gmail doesn't consider any apps that send mail using SMTP protocol secure. When you try and run the app with a Gmail account you'll get security warnings on that account unless you have enabled what Google calls ["Less Secure Apps"](https://support.google.com/accounts/answer/6010255?hl=en).
+
 ## Testing the app frontend
 
 `pytest` should run all the tests in the `tests` folder.
\ No newline at end of file
diff --git a/parser/__init__.py b/parser/__init__.py
index 4d2ad07..09cc56c 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -1,8 +1,18 @@
 import os
+import sys
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import settings
 
+from celery import Celery
 from flask import Flask
+from flask_mail import Mail
+
+# redis_client = FlaskRedis(app)
 
+celery = Celery(__name__, broker=settings.CELERY_BROKER_URL, result_backend=settings.CELERY_RESULT_BACKEND)
+mail = Mail()
 
 def create_app(test_config=None):
     # create and configure the app
@@ -14,23 +24,35 @@ def create_app(test_config=None):
         MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
         UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Input_Data'),
         RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_Data'),
-        SCRIPTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Code')
-        # CELERY_BROKER_URL=,
-        # CELERY_RESULT_BACKEND=
+        SCRIPTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Code'),
+        CELERY_BROKER_URL=settings.CELERY_BROKER_URL,
+        CELERY_RESULT_BACKEND=settings.CELERY_RESULT_BACKEND,
+        MAIL_SERVER = settings.MAIL_SERVER,
+        MAIL_PORT = settings.MAIL_PORT,
+        MAIL_USE_TLS = True,
+        MAIL_USERNAME = settings.MAIL_USERNAME,
+        MAIL_PASSWORD = settings.MAIL_PASSWORD,
+        EMAIL_SENDER = settings.EMAIL_SENDER,
+        EMAIL_SUBJECT = settings.EMAIL_SUBJECT,
+        BASE_URL = settings.BASE_URL
     )
 
-    from . import db
+    # redis_client.init_app(app)
+    import db
     db.init_app(app)
 
-    from . import main
+    import main
     app.register_blueprint(main.bp)
 
-    if test_config is None:
-        # load the instance config, if it exists, when not testing
-        app.config.from_pyfile('config.py', silent=True)
-    else:
-        # load the test config if passed in
-        app.config.from_mapping(test_config)
+    import tasks
+    app.register_blueprint(tasks.bp)
+
+    # if test_config is None:
+    #     # load the instance config, if it exists, when not testing
+    #     app.config.from_pyfile('config.py', silent=True)
+    # else:
+    #     # load the test config if passed in
+    #     app.config.from_mapping(test_config)
 
     # ensure the instance folder exists
     try:
@@ -38,4 +60,7 @@ def create_app(test_config=None):
     except OSError:
         pass
 
+    celery.conf.update(app.config)
+    mail.init_app(app)
+
     return app
\ No newline at end of file
diff --git a/parser/celery_worker.py b/parser/celery_worker.py
new file mode 100644
index 0000000..18e35bc
--- /dev/null
+++ b/parser/celery_worker.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+import os
+from __init__ import celery, create_app
+
+app = create_app(os.getenv('FLASK_CONFIG') or 'default')
+app.app_context().push()
\ No newline at end of file
diff --git a/parser/main.py b/parser/main.py
index 27a16bb..66c47c4 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -10,7 +10,7 @@
 
 from flask import (Blueprint, current_app, flash, Flask, g, Markup,
     redirect, render_template, request, send_from_directory, url_for)
-from . import db
+import db
 from werkzeug.utils import secure_filename
 
 bp = Blueprint('main', __name__, url_prefix='')
diff --git a/requirements.txt b/requirements.txt
index 94b8308..1ddc4a7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 celery==5.0.5
 coverage==5.4
 Flask==1.1.2
+Flask-Mail==0.9.1
 matplotlib==3.3.3
 nltk==3.5
 numpy==1.19.4
@@ -8,6 +9,7 @@ pandas==1.1.5
 pathlib==1.0.1
 pytest==6.2.2
 python-Levenshtein==0.12.0
+redis==3.5.3
 scikit-learn==0.23.2
 scipy==1.5.4
 spacy==2.3.4
diff --git a/settings.py.example b/settings.py.example
index 2a179d4..83d365b 100644
--- a/settings.py.example
+++ b/settings.py.example
@@ -1,2 +1,12 @@
 SECRET_KEY = '' # keep this secret in production!
 MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16 MB
+CELERY_BROKER_URL = 'redis://localhost:6379/0'
+CELERY_RESULT_BACKEND = 'redis://localhost:6379/1'
+EMAIL_SENDER = 'Votetripling SMS Transcript Processing, vtsms@example.com'
+BASE_URL = 'https://www.example.com/'
+MAIL_SERVER = 'smtp.gmail.com'
+MAIL_PORT = 587
+MAIL_USE_TLS = True
+MAIL_USERNAME = 'somethingsomething@gmail.com'
+MAIL_PASSWORD = '1234whataterriblepassword'
+TESTING = False
\ No newline at end of file

From 989b59106923521e337aaba2f247b237ba02bb8a Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 22 Apr 2021 12:45:37 -0600
Subject: [PATCH 36/52] switched to instance relative config

---
 .gitignore                 |  5 +++--
 README_local.md            |  9 ++++++--
 instance/config.py.example | 21 +++++++++++++++++++
 parser/__init__.py         | 43 +++++++++++++-------------------------
 settings.py.example        | 12 -----------
 5 files changed, 45 insertions(+), 45 deletions(-)
 create mode 100644 instance/config.py.example
 delete mode 100644 settings.py.example

diff --git a/.gitignore b/.gitignore
index faf19dc..73442c5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,11 +2,12 @@
 *.csv
 *.xlsx
 settings.py
-
-instance/
+config.py
 
 __pycache__
 *.pyc
+*.rdb
+*.sqlite
 
 .pytest_cache/
 .coverage
diff --git a/README_local.md b/README_local.md
index 90edbc7..c86c4fd 100644
--- a/README_local.md
+++ b/README_local.md
@@ -130,7 +130,7 @@ This script will output two files:
 - *names_extract* a guess for the extracted names (to be reviewed)
 
 # Running the app frontend
-app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and running the scripts with them. 
+app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and requesting that the above scripts be run with them. It uses Celery workers, a Redis queue and Flask-mail to manage script jobs and email notifications with the results in the background.
 Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
 
 You'll need to install Redis. On OSX, install homebrew and then `brew install redis`. You may also need to run `pip install "celery[redis]"`
@@ -146,12 +146,17 @@ That will create a file named `parser.sqlite3` (the application database) in the
 `
 and access the running application at [http://localhost:5000/](http://localhost:5000/)
 
-## Configuring and testing email sending
+## Configuring email
 
 Email config variables in the example config file assume you are using Gmail for testing. Two important notes:
 * Gmail probably isn't adequate for production scale; you can only send about 100 emails a day.
 * Gmail doesn't consider any apps that send mail using SMTP protocol secure. When you try and run the app with a Gmail account you'll get security warnings on that account unless you have enabled what Google calls ["Less Secure Apps"](https://support.google.com/accounts/answer/6010255?hl=en).
 
+## Background tasks
+
+* `celery -A celery_worker.celery worker --loglevel=info` will spin up a celery worker for you in a local dev environment.
+* Run redis in a different terminal window with `redis-server`.
+
 ## Testing the app frontend
 
 `pytest` should run all the tests in the `tests` folder.
\ No newline at end of file
diff --git a/instance/config.py.example b/instance/config.py.example
new file mode 100644
index 0000000..d25fc23
--- /dev/null
+++ b/instance/config.py.example
@@ -0,0 +1,21 @@
+import os
+
+SECRET_KEY = 'dev' # change this for prod!
+BASE_URL = 'http://localhost:5000/' # local dev
+MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16 MB
+
+APP_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+UPLOAD_FOLDER = os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Input_Data'),
+RESULTS_FOLDER = os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_Data'),
+SCRIPTS_FOLDER = os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Code'),
+
+CELERY_BROKER_URL = 'redis://localhost:6379/0'
+CELERY_RESULT_BACKEND = 'redis://localhost:6379/1'
+
+MAIL_SERVER = 'smtp.gmail.com'
+MAIL_PORT = 587
+MAIL_USE_TLS = True
+MAIL_USERNAME = 'notarealemail@gmail.com'
+MAIL_PASSWORD = 'password1234'
+EMAIL_SENDER = 'Votetripling SMS Transcript Processing, notarealemail@gmail.com'
+EMAIL_SUBJECT = 'SMS transcript processing'
diff --git a/parser/__init__.py b/parser/__init__.py
index 09cc56c..6123937 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -2,42 +2,33 @@
 import sys
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import settings
+sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'instance'))
 
 from celery import Celery
 from flask import Flask
 from flask_mail import Mail
+import config
 
 # redis_client = FlaskRedis(app)
 
-celery = Celery(__name__, broker=settings.CELERY_BROKER_URL, result_backend=settings.CELERY_RESULT_BACKEND)
+celery = Celery(__name__, broker=config.CELERY_BROKER_URL, result_backend=config.CELERY_RESULT_BACKEND)
 mail = Mail()
 
 def create_app(test_config=None):
     # create and configure the app
     app = Flask(__name__, instance_relative_config=True)
-    APP_ROOT = os.path.dirname(os.path.abspath(__file__))
-    app.config.from_mapping(
-        SECRET_KEY=settings.SECRET_KEY,
-        DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
-        MAX_CONTENT_LENGTH=settings.MAX_CONTENT_LENGTH,
-        UPLOAD_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Input_Data'),
-        RESULTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_Data'),
-        SCRIPTS_FOLDER=os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Code'),
-        CELERY_BROKER_URL=settings.CELERY_BROKER_URL,
-        CELERY_RESULT_BACKEND=settings.CELERY_RESULT_BACKEND,
-        MAIL_SERVER = settings.MAIL_SERVER,
-        MAIL_PORT = settings.MAIL_PORT,
-        MAIL_USE_TLS = True,
-        MAIL_USERNAME = settings.MAIL_USERNAME,
-        MAIL_PASSWORD = settings.MAIL_PASSWORD,
-        EMAIL_SENDER = settings.EMAIL_SENDER,
-        EMAIL_SUBJECT = settings.EMAIL_SUBJECT,
-        BASE_URL = settings.BASE_URL
+    test_config = None
+    if test_config is None:
+        # load the instance config, if it exists, when not testing
+        app.config.from_pyfile('config.py', silent=True)
+    else:
+        # load the test config if passed in
+        app.config.from_mapping(test_config)
+
+    app.config.update(
+        DATABASE = os.path.join(app.instance_path, 'parser.sqlite'),
     )
 
-    # redis_client.init_app(app)
     import db
     db.init_app(app)
 
@@ -47,19 +38,13 @@ def create_app(test_config=None):
     import tasks
     app.register_blueprint(tasks.bp)
 
-    # if test_config is None:
-    #     # load the instance config, if it exists, when not testing
-    #     app.config.from_pyfile('config.py', silent=True)
-    # else:
-    #     # load the test config if passed in
-    #     app.config.from_mapping(test_config)
-
     # ensure the instance folder exists
     try:
         os.makedirs(app.instance_path)
     except OSError:
         pass
 
+    # redis_client.init_app(app)
     celery.conf.update(app.config)
     mail.init_app(app)
 
diff --git a/settings.py.example b/settings.py.example
deleted file mode 100644
index 83d365b..0000000
--- a/settings.py.example
+++ /dev/null
@@ -1,12 +0,0 @@
-SECRET_KEY = '' # keep this secret in production!
-MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16 MB
-CELERY_BROKER_URL = 'redis://localhost:6379/0'
-CELERY_RESULT_BACKEND = 'redis://localhost:6379/1'
-EMAIL_SENDER = 'Votetripling SMS Transcript Processing, vtsms@example.com'
-BASE_URL = 'https://www.example.com/'
-MAIL_SERVER = 'smtp.gmail.com'
-MAIL_PORT = 587
-MAIL_USE_TLS = True
-MAIL_USERNAME = 'somethingsomething@gmail.com'
-MAIL_PASSWORD = '1234whataterriblepassword'
-TESTING = False
\ No newline at end of file

From fc755c5736788d80d1dfe336d197365d9a747d38 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 23 Apr 2021 16:17:15 -0600
Subject: [PATCH 37/52] moved test target email to config

---
 instance/config.py.example |  1 +
 parser/__init__.py         | 14 +++++---------
 parser/main.py             |  1 -
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/instance/config.py.example b/instance/config.py.example
index d25fc23..c25a779 100644
--- a/instance/config.py.example
+++ b/instance/config.py.example
@@ -19,3 +19,4 @@ MAIL_USERNAME = 'notarealemail@gmail.com'
 MAIL_PASSWORD = 'password1234'
 EMAIL_SENDER = 'Votetripling SMS Transcript Processing, notarealemail@gmail.com'
 EMAIL_SUBJECT = 'SMS transcript processing'
+TEST_TARGET_EMAIL = 'your.email@example.com'
diff --git a/parser/__init__.py b/parser/__init__.py
index 6123937..2fcc105 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -6,17 +6,18 @@
 
 from celery import Celery
 from flask import Flask
-from flask_mail import Mail
 import config
 
-# redis_client = FlaskRedis(app)
-
 celery = Celery(__name__, broker=config.CELERY_BROKER_URL, result_backend=config.CELERY_RESULT_BACKEND)
-mail = Mail()
 
 def create_app(test_config=None):
     # create and configure the app
     app = Flask(__name__, instance_relative_config=True)
+    app.instance_path = (os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'instance')) # maybe setting this manually fixes the config problem in blueprint celery tasks
+    app.config.from_mapping(
+            SECRET_KEY='dev',
+            DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
+        )
     test_config = None
     if test_config is None:
         # load the instance config, if it exists, when not testing
@@ -25,10 +26,6 @@ def create_app(test_config=None):
         # load the test config if passed in
         app.config.from_mapping(test_config)
 
-    app.config.update(
-        DATABASE = os.path.join(app.instance_path, 'parser.sqlite'),
-    )
-
     import db
     db.init_app(app)
 
@@ -46,6 +43,5 @@ def create_app(test_config=None):
 
     # redis_client.init_app(app)
     celery.conf.update(app.config)
-    mail.init_app(app)
 
     return app
\ No newline at end of file
diff --git a/parser/main.py b/parser/main.py
index 66c47c4..cf1cb89 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -3,7 +3,6 @@
 import functools
 import os
 import random
-import settings
 import sqlite3
 import string
 import subprocess

From fbbd52dddabe91b187e66220387394f49359a53b Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Fri, 23 Apr 2021 16:20:57 -0600
Subject: [PATCH 38/52] readd tasks file

---
 parser/tasks.py | 126 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 126 insertions(+)
 create mode 100644 parser/tasks.py

diff --git a/parser/tasks.py b/parser/tasks.py
new file mode 100644
index 0000000..b3e1a8f
--- /dev/null
+++ b/parser/tasks.py
@@ -0,0 +1,126 @@
+""" 
+    TODO:
+    Production implementation should process jobs and 
+    send emails with a queue and workers.
+"""
+from __init__ import celery
+from flask import Blueprint, current_app, request
+from flask_mail import Mail, Message
+from main import UPLOAD_TYPES, unique_filename
+import config
+import redis
+
+
+bp = Blueprint('tasks', __name__, url_prefix='/tasks')
+
+@celery.task
+def run_script(input_file, job_type, email):
+
+    # It seems silly to run these via the command line
+    # but we can easily change this later. Trying to avoid
+    # messing with scripts unless we absolutely have to.
+    scripts_folder = config.SCRIPTS_FOLDER
+    scripts_home_dir = os.path.dirname(scripts_folder)
+    output_file = unique_filename()
+    second_output_file = None
+    if job_type == 'vec_file':
+        second_output_file = unique_filename()
+
+    if job_type == 'tblc_file':
+        cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file)
+    elif job_type == 'vec_file':
+        cmd = 'python {}/van_export_cleaning.py -i {} -f {} -o {} -m {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file, second_output_file)
+    elif job_type == 'tblc_tmc_file':
+        cmd = 'python {}/name_cleaning_with_responses.py -i {} -f {} -o {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file)
+    elif job_type == 'sccne_file':
+        cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file)
+    # elif job_type == 'sms_agg_file':
+        # this script is to be ported to Python
+    else:
+        print("Unknown job type {}".format(job_type))
+    print("cmd", cmd)
+    job_run = subprocess.run(cmd, stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT, shell=True)
+    if job_run.returncode: # exit code of 0 is success, 1 is generic error
+        status = 'error'
+        err_log = job_run.stdout.decode()
+        print("ERROR: Could not process job {} file {} type {}".format(job_id, input_file, job_type))
+        print(err_log)
+    else:
+        print("SUCCESS")
+    result_file = None
+    if status == 'success':
+        result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], output_file)
+        if second_output_file:
+            result_file == '{0}/{1}|{0}/{2}'.format(
+                current_app.config['RESULTS_FOLDER'], output_file, second_output_file)
+    done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
+    update = db.query_db(done_query, (status, result_file, job_id), write=True)
+    if status == 'success':
+        return True, output_file, second_output_file
+    return False, err_log, None
+    pass
+
+
+@celery.task
+def email_results(data):
+    with current_app.app_context():
+        # for some reason celery spins up a new instance without access to 
+        # original config file, so we re-config here
+        current_app.config.from_object(config)
+        mail = Mail(current_app)
+        msg = Message(config.EMAIL_SUBJECT,
+                sender=config.EMAIL_SENDER,
+                recipients=[data['email']])
+        # accommodating more than one output file
+        output = data['result_file'].split('|')
+        results = ''.join(['<p>{}results/{}</p>'.format(config.BASE_URL, output_file) for output_file in output])
+        msg.html = (
+            "<p>Thank you for using the votetripling.org SMS transcript processing tool.</p>"
+            "<p>The {} script successfully processed your data file.</p>"
+            "<p>Link(s) to download the results:</p>"
+            "{}"
+            "<p>Your result files will be available for download for 48 hours.</p>"
+        ).format(UPLOAD_TYPES[data['upload_type']]['name'], results)
+        mail.send(msg)
+
+
+@bp.route('/email-results/', methods=['GET'])
+def index():
+    if request.method == 'GET':
+        # TODO: get real job data
+        data = {
+            'job_id': 1234,
+            'upload_type': 'vec_file',
+            'result_file': '123.csv|456.csv',
+            'email': current_app.config['TEST_TARGET_EMAIL']
+        }
+        res = email_results.apply_async(args=[data], countdown=0)
+        msg = ('Email queued for send! Check your inbox (and spam) for an email '
+            'from "{}" with the subject "{}".').format(
+                current_app.config['EMAIL_SENDER'], current_app.config['EMAIL_SUBJECT'])
+        return msg
+    return 'not allowed here'
+
+@bp.route('/script/', methods=['GET'])
+def index():
+    if request.method == 'GET':
+        # TODO: get real job data
+        msg = ('Script queued for processing')
+        return msg
+    return 'not allowed here'
+
+def cleanup_files(filename):
+    # TODO
+    # delete files from finished jobs that are more than 48 hours old
+    pass
+
+
+def cleanup_jobs(interval):
+    # TODO
+    # delete jobs that are more than `interval` old, to keep the db from getting too big
+    pass
\ No newline at end of file

From 0d7b0e5992c7d970e8f39abe04dabbbc7d139a43 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Tue, 27 Apr 2021 16:29:02 -0600
Subject: [PATCH 39/52] enabled sms aggregation script with no interactivity

---
 instance/config.py.example        |  2 +
 parser/__init__.py                |  4 +-
 parser/main.py                    | 18 +++++---
 parser/tasks.py                   | 71 +++++++++++++++++--------------
 parser/templates/upload_form.html | 11 ++++-
 5 files changed, 67 insertions(+), 39 deletions(-)

diff --git a/instance/config.py.example b/instance/config.py.example
index c25a779..d01c7b2 100644
--- a/instance/config.py.example
+++ b/instance/config.py.example
@@ -20,3 +20,5 @@ MAIL_PASSWORD = 'password1234'
 EMAIL_SENDER = 'Votetripling SMS Transcript Processing, notarealemail@gmail.com'
 EMAIL_SUBJECT = 'SMS transcript processing'
 TEST_TARGET_EMAIL = 'your.email@example.com'
+
+FILE_LIFE = 72 # no. of hrs we let uploaded and result files hang out on the server
diff --git a/parser/__init__.py b/parser/__init__.py
index 2fcc105..ea89b30 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -1,3 +1,5 @@
+# because this code gets run from several different places, update PATH
+# so we can find modules from wherever we run things
 import os
 import sys
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
@@ -16,7 +18,7 @@ def create_app(test_config=None):
     app.instance_path = (os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'instance')) # maybe setting this manually fixes the config problem in blueprint celery tasks
     app.config.from_mapping(
             SECRET_KEY='dev',
-            DATABASE=os.path.join(app.instance_path, 'parser.sqlite'),
+            DATABASE=os.path.join(app.instance_path, 'parser.sqlite')
         )
     test_config = None
     if test_config is None:
diff --git a/parser/main.py b/parser/main.py
index cf1cb89..d416be9 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -7,9 +7,14 @@
 import string
 import subprocess
 
+import config
+import db
+
+from __init__ import celery
 from flask import (Blueprint, current_app, flash, Flask, g, Markup,
     redirect, render_template, request, send_from_directory, url_for)
-import db
+from flask_mail import Mail, Message
+
 from werkzeug.utils import secure_filename
 
 bp = Blueprint('main', __name__, url_prefix='')
@@ -102,22 +107,25 @@ def process_job():
     if job_type == 'vec_file':
         second_output_file = unique_filename()
 
+    cmd = None
     if job_type == 'tblc_file':
         cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
     elif job_type == 'vec_file':
         cmd = 'python {}/van_export_cleaning.py -i {} -f {} -o {} -m {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file, second_output_file)
-    elif job_type == 'tblc_tmc_file':
+    elif job_type == 'tblctmc_file':
         cmd = 'python {}/name_cleaning_with_responses.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
     elif job_type == 'sccne_file':
         cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
-    # elif job_type == 'sms_agg_file':
-        # this script is to be ported to Python
+    elif job_type == 'smsagg_file':
+        cmd = 'python {}/aggregate_text_messages.py -d {} -o {}/{}'.format(
+            scripts_folder, input_file, current_app.config['RESULTS_FOLDER'], output_file)
     else:
-        print("Unknown job type {}".format(job_type))
+        err_log = "Unknown job type {}".format(job_type)
+        return False, err_log, None
     print("cmd", cmd)
     job_run = subprocess.run(cmd, stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT, shell=True)
diff --git a/parser/tasks.py b/parser/tasks.py
index b3e1a8f..4bf5c3c 100644
--- a/parser/tasks.py
+++ b/parser/tasks.py
@@ -8,47 +8,49 @@
 from flask_mail import Mail, Message
 from main import UPLOAD_TYPES, unique_filename
 import config
-import redis
 
 
 bp = Blueprint('tasks', __name__, url_prefix='/tasks')
 
 @celery.task
-def run_script(input_file, job_type, email):
-
+def process_job_async(data):
     # It seems silly to run these via the command line
     # but we can easily change this later. Trying to avoid
     # messing with scripts unless we absolutely have to.
+    input_file = data['input_file']
+    upload_type = data['upload_type']
+    email = data['email']
     scripts_folder = config.SCRIPTS_FOLDER
     scripts_home_dir = os.path.dirname(scripts_folder)
     output_file = unique_filename()
     second_output_file = None
-    if job_type == 'vec_file':
+    if upload_type == 'vec_file':
         second_output_file = unique_filename()
 
-    if job_type == 'tblc_file':
+    if upload_type == 'tblc_file':
         cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
-    elif job_type == 'vec_file':
+    elif upload_type == 'vec_file':
         cmd = 'python {}/van_export_cleaning.py -i {} -f {} -o {} -m {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file, second_output_file)
-    elif job_type == 'tblc_tmc_file':
+    elif upload_type == 'tblctmc_file':
         cmd = 'python {}/name_cleaning_with_responses.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
-    elif job_type == 'sccne_file':
+    elif upload_type == 'sccne_file':
         cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
-    # elif job_type == 'sms_agg_file':
-        # this script is to be ported to Python
+    elif upload_type == 'smsagg_file':
+        cmd = 'python {}/aggregate_text_messages.py -d {} -o {}'.format(
+            scripts_folder, input_file, output_file)
     else:
-        print("Unknown job type {}".format(job_type))
+        print("Unknown job type {}".format(upload_type))
     print("cmd", cmd)
     job_run = subprocess.run(cmd, stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT, shell=True)
     if job_run.returncode: # exit code of 0 is success, 1 is generic error
         status = 'error'
         err_log = job_run.stdout.decode()
-        print("ERROR: Could not process job {} file {} type {}".format(job_id, input_file, job_type))
+        print("ERROR: Could not process job {} file {} type {}".format(job_id, input_file, upload_type))
         print(err_log)
     else:
         print("SUCCESS")
@@ -58,13 +60,16 @@ def run_script(input_file, job_type, email):
         if second_output_file:
             result_file == '{0}/{1}|{0}/{2}'.format(
                 current_app.config['RESULTS_FOLDER'], output_file, second_output_file)
-    done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
-    update = db.query_db(done_query, (status, result_file, job_id), write=True)
     if status == 'success':
+        data = {
+            'result_file': result_file,
+            'email': email or config.TEST_TARGET_EMAIL,
+            'upload_type': upload_type
+        }
+        res = email_results.apply_async(args=[data], countdown=0)
+        # also delete the input file
         return True, output_file, second_output_file
     return False, err_log, None
-    pass
-
 
 @celery.task
 def email_results(data):
@@ -84,15 +89,16 @@ def email_results(data):
             "<p>The {} script successfully processed your data file.</p>"
             "<p>Link(s) to download the results:</p>"
             "{}"
-            "<p>Your result files will be available for download for 48 hours.</p>"
-        ).format(UPLOAD_TYPES[data['upload_type']]['name'], results)
+            "<p>Your result files will be available for download for {} hours.</p>"
+        ).format(UPLOAD_TYPES[data['upload_type']]['name'], results, config.FILE_LIFE)
         mail.send(msg)
+    return True
 
 
+# for testing only
 @bp.route('/email-results/', methods=['GET'])
-def index():
+def email():
     if request.method == 'GET':
-        # TODO: get real job data
         data = {
             'job_id': 1234,
             'upload_type': 'vec_file',
@@ -100,27 +106,30 @@ def index():
             'email': current_app.config['TEST_TARGET_EMAIL']
         }
         res = email_results.apply_async(args=[data], countdown=0)
+        if res.failed():
+            print('emailing results failed')
         msg = ('Email queued for send! Check your inbox (and spam) for an email '
             'from "{}" with the subject "{}".').format(
                 current_app.config['EMAIL_SENDER'], current_app.config['EMAIL_SUBJECT'])
         return msg
     return 'not allowed here'
 
-@bp.route('/script/', methods=['GET'])
-def index():
+# to manually kick off processing - note that this doesn't take an email
+@bp.route('/process/<upload_type>/<file_name>', methods=['GET'])
+def process():
     if request.method == 'GET':
-        # TODO: get real job data
-        msg = ('Script queued for processing')
+        data = {
+            'input_file': '{}/file_name'.format(config.UPLOAD_FOLDER),
+            'upload_type': upload_type,
+            'email': None
+        }
+        res = process_job_async.apply_async(args=[data], countdown=0)
+        msg = ('File {} queued for processing as {}. Results will be emailed to {}.').format(
+            file_name, upload_type, config.TEST_TARGET_EMAIL)
         return msg
     return 'not allowed here'
 
 def cleanup_files(filename):
     # TODO
-    # delete files from finished jobs that are more than 48 hours old
+    # delete files from finished jobs that are more than n hours old
     pass
-
-
-def cleanup_jobs(interval):
-    # TODO
-    # delete jobs that are more than `interval` old, to keep the db from getting too big
-    pass
\ No newline at end of file
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index e89e43a..c536677 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -168,13 +168,20 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
 
         </div>
 
-        <div class="upload-disabled">
+        <div class="upload-enabled">
             <hr />
 
             <h2 id="sms_agg">SMS Aggregation</h2>
             <blockquote>I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.</blockquote>
 
-            <p>Upload a csv file of raw individual SMS messages, not grouped by conversation.</p>
+            <p>Upload a csv file of raw individual SMS messages. The csv should contain the following columns:</p>
+            <ul>
+                <li><strong>MessageId:</strong> Message timestamp OR an ascending unique ID for the message, used to determine message order</li>
+                <li><strong>ConversationId:</strong> unique ID for each conversation</li>
+                <li><strong>MessageDirection:</strong> direction ("inbound" or "outbound") of each message</li>
+                <li><strong>MessageBody:</strong> the body/content of the message</li>
+                <li><strong>EndpointPhoneNumber:</strong> phone number or any other unique identifier for the recipient</li>
+            </ul>
 
             <form method="post" enctype="multipart/form-data">
                 <div>

From c444955ac0fc36cd8319b8c61cccdc82be255601 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 28 Apr 2021 12:48:55 -0600
Subject: [PATCH 40/52] extraction has three outputs, not two

---
 README_local.md                   |   8 +-
 parser/main.py                    |  29 +++--
 parser/tasks.py                   |  10 +-
 parser/templates/upload_form.html | 199 ++++++++++++++++--------------
 4 files changed, 134 insertions(+), 112 deletions(-)

diff --git a/README_local.md b/README_local.md
index c86c4fd..501d53a 100644
--- a/README_local.md
+++ b/README_local.md
@@ -58,7 +58,7 @@ Add a CSV to the Input_Data folder. This csv file must be of the same format as
 In this directory, run `python3 Code/annotate_conversations.py -i [input_filename]`.
 
 **Outputs:**
-This script will output two files:
+This script will output three files:
 1. A file of triplers called `sms_triplers.csv`. For each tripler, we provide the following fields (each row represents one text message conversation):
 - *ConversationId* a unique identifier for the conversation
 - *contact_phone* the phone number of the target 
@@ -78,6 +78,7 @@ This script will output two files:
 - *wrong_number* guess for did we have the wrong number for this person (to be reviewed)
 - *names_extract* guess for what names (if any) were provided by this person as tripling targets (to be reviewed)
 
+3. A file of opt-outs
 
 ## Text Banker Log Cleaning
 **Use Case:** I have text banker logs for names provided by vote triplers. I need these logs cleaned up and standardized.
@@ -130,11 +131,14 @@ This script will output two files:
 - *names_extract* a guess for the extracted names (to be reviewed)
 
 # Running the app frontend
-app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and requesting that the above scripts be run with them. It uses Celery workers, a Redis queue and Flask-mail to manage script jobs and email notifications with the results in the background.
+app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and requesting that the above scripts be run with them. If config.PROCESS_ASYNC is set to True, it uses Celery workers, a Redis queue and Flask-mail to manage script jobs and email notifications with the results in the background. If config.PROCESS_ASYNC is set to False, it runs script jobs synchronously from a queue/log stored in a SQLite database.
+
 Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
 
 You'll need to install Redis. On OSX, install homebrew and then `brew install redis`. You may also need to run `pip install "celery[redis]"`
 
+Copy config.py from `instance/config.py.example` file and fill it in.
+
 To run an instance of the frontend locally, from the project root directory initialize the db:
 `
 export FLASK_APP=parser
diff --git a/parser/main.py b/parser/main.py
index d416be9..adfd172 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -104,9 +104,11 @@ def process_job():
     print('shd', scripts_home_dir)
     output_file = unique_filename()
     second_output_file = None
-    if job_type == 'vec_file':
+    third_output_file = None
+    if job_type in ['vec_file','sccne_file']:
         second_output_file = unique_filename()
-
+        if job_type == 'sccne_file':
+            third_output_file = unique_filename()
     cmd = None
     if job_type == 'tblc_file':
         cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
@@ -118,8 +120,9 @@ def process_job():
         cmd = 'python {}/name_cleaning_with_responses.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
     elif job_type == 'sccne_file':
-        cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {}'.format(
-            scripts_folder, input_file, scripts_home_dir, output_file)
+        cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {} -n {} -m {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file, second_output_file,
+            third_output_file)
     elif job_type == 'smsagg_file':
         cmd = 'python {}/aggregate_text_messages.py -d {} -o {}/{}'.format(
             scripts_folder, input_file, current_app.config['RESULTS_FOLDER'], output_file)
@@ -140,13 +143,16 @@ def process_job():
     if status == 'success':
         result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], output_file)
         if second_output_file:
-            result_file == '{0}/{1}|{0}/{2}'.format(
+            result_file = '{0}/{1}|{0}/{2}'.format(
                 current_app.config['RESULTS_FOLDER'], output_file, second_output_file)
+        if third_output_file:
+            result_file = '{0}/{1}|{0}/{2}|{0}/{3}'.format(
+                current_app.config['RESULTS_FOLDER'], output_file, second_output_file, third_output_file)
     done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
     update = db.query_db(done_query, (status, result_file, job_id), write=True)
     if status == 'success':
-        return True, output_file, second_output_file
-    return False, err_log, None
+        return True, output_file, second_output_file, third_output_file
+    return False, err_log, None, None
 
 
 def email_results():
@@ -204,15 +210,16 @@ def index():
             # queue_msg = ('Processing file {} as {}'.format(
             #                 filename, UPLOAD_TYPES[upload_type]['name']))
             # flash(queue_msg, 'info')
-            success, output, second_output = process_job()
+            success, output, second_output, third_output = process_job()
+            output_files = [output, second_output, third_output]
             if success:
                 outcome_msg = Markup(
                     '<a href="/results/{}">Download results</a>'.format(output))
                 if second_output:
+                    result_links = ['<a href="/results/{}">file {}</a>'.format(
+                        x, output_files.index(x) + 1) for x in output_files]
                     outcome_msg = Markup(
-                        'Download <a href="/results/{}">result file 1</a>'
-                        ' and <a href="/results/{}">result file 2</a>').format(
-                            output, second_output)
+                        'Download results:  {}'.format('  '.join(result_links)))
             else:
                 outcome_msg = 'Error processing file {}'.format(
                     output) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
diff --git a/parser/tasks.py b/parser/tasks.py
index 4bf5c3c..ef6793a 100644
--- a/parser/tasks.py
+++ b/parser/tasks.py
@@ -24,8 +24,11 @@ def process_job_async(data):
     scripts_home_dir = os.path.dirname(scripts_folder)
     output_file = unique_filename()
     second_output_file = None
-    if upload_type == 'vec_file':
+    third_output_file = None
+    if job_type in ['vec_file','sccne_file']:
         second_output_file = unique_filename()
+        if job_type == 'sccne_file':
+            third_output_file = unique_filename()
 
     if upload_type == 'tblc_file':
         cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
@@ -37,8 +40,9 @@ def process_job_async(data):
         cmd = 'python {}/name_cleaning_with_responses.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
     elif upload_type == 'sccne_file':
-        cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {}'.format(
-            scripts_folder, input_file, scripts_home_dir, output_file)
+        cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {} -n {} -m {}'.format(
+            scripts_folder, input_file, scripts_home_dir, output_file,
+            second_output_file, third_output_file)
     elif upload_type == 'smsagg_file':
         cmd = 'python {}/aggregate_text_messages.py -d {} -o {}'.format(
             scripts_folder, input_file, output_file)
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index c536677..004eb45 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -60,14 +60,115 @@
             <h1>Vote tripling SMS transcript parsing</h1>
 
             <ul class="menu">
+                <li><a href="#sms_agg">SMS Aggregation</a></li>
+                <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
                 <li><a href="#text_banker_log_cleaning">Text banker log cleaning</a></li>
                 <li><a href="#text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (utilizing text message conversation)</a></li>
                 <li><a href="#van_export_cleaning">VAN export cleaning</a></li>
-                <li><a href="#sms_agg" class="disabled">SMS Aggregation</a></li>
-                <li><a href="#sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</a></li>
             </ul>
         </div>
 
+        <div class="upload-enabled">
+            <hr />
+
+            <h2 id="sms_agg">SMS Aggregation</h2>
+            <blockquote>I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.</blockquote>
+
+            <p>Upload a csv file of raw individual SMS messages. The csv should contain the following columns:</p>
+            <ul>
+                <li><strong>MessageId:</strong> Message timestamp OR an ascending unique ID for the message, used to determine message order</li>
+                <li><strong>ConversationId:</strong> unique ID for each conversation</li>
+                <li><strong>MessageDirection:</strong> direction ("inbound" or "outbound") of each message</li>
+                <li><strong>MessageBody:</strong> the body/content of the message</li>
+                <li><strong>EndpointPhoneNumber:</strong> phone number or any other unique identifier for the recipient</li>
+            </ul>
+
+            <form method="post" enctype="multipart/form-data">
+                <div>
+                    <label for="smsagg_email">Your email:</label>
+                    <input type="email" id="smsagg_email" name="smsagg_email" size="30"/>
+                </div>
+                <div>
+                    <label for="smsagg_file">Your CSV:</label>
+                    <input type="file" id="smsagg_file" name="smsagg_file" accept=".csv" />
+                    <input type="submit" value="Upload" />
+                </div>
+            </form>
+
+            <p id="sms_agg_output">If successful, the output will be a CSV with a single row representing each text message conversation, including the following fields:
+            <ul>
+                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                <li><strong>contact_phone:</strong> the phone number of the target</li>
+                <li><strong>totalMessages:</strong> the total number of messages exchanged</li>
+                <li><strong>tripleMessage:</strong> initial message sent from the text banker to the target</li>
+                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                <li><strong>tripleResponse:</strong> follow up message sent from the text banker to the target</li>
+                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                <li><strong>tripleFinal:</strong> final follow up sent by text banker</li>
+                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                <li><strong>noResponse:</strong> boolean for whether there was no response</li>
+                <li><strong>negResponse:</strong> boolean for generally negative or discouraging terms (sorry, no, etc.)</li>
+                <li><strong>posResponse:</strong> boolean for generally positive or encouraging terms</li>
+                <li><strong>affirmResponse:</strong> boolean for presence of a scripted affirmation by text banker</li>
+                <li><strong>finalAffirmResponse:</strong> boolean for presence of a scripted follow up affirmation by text banker</li>
+            </ul>
+
+        </div>
+
+        <div class="upload-enabled">
+            <hr />
+            <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
+            <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
+
+            <p>Upload a csv file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a> with a column of unique IDs labeled <strong>conversationid</strong>.</p>
+            
+            <form method="post" enctype="multipart/form-data">
+                <div>
+                    <label for="sccne_email">Your email:</label>
+                    <input type="email" id="sccne_email" name="sccne_email" size="30"/>
+                </div>
+                <div>
+                    <label for="sccne_file">Your CSV:</label>
+                    <input type=file id="sccne_file" name="sccne_file" accept=".csv">
+                    <input type="submit" value="Upload"/>
+                </div>
+            </form>
+
+            <p>If successful, the output will include two files:
+            <ul>
+                <li>A file of triplers. For each tripler, we provide the following fields (each row represents one text message conversation):
+                <ul>
+                    <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                    <li><strong>contact_phone:</strong> the phone number of the target</li>
+                    <li><strong>is_tripler:</strong> did this person agree to be a tripler ('yes' for everyone in this file)</li>
+                    <li><strong>opted_out:</strong> did this person opt out of future messages</li>
+                    <li><strong>wrong_number:</strong> did we have the wrong number for this person</li>
+                    <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
+                </ul>
+
+                <li>A file of conversations for manual review, with the following fields:
+                <ul>
+                    <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
+                    <li><strong>contact_phone:</strong> the phone number of the target</li>
+                    <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
+                    <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
+                    <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
+                    <li><strong>is_tripler:</strong> guess for did this person agree to be a tripler (to be reviewed)</li>
+                    <li><strong>opted_out:</strong> guess for did this person opt out of future messages (to be reviewed)</li>
+                    <li><strong>wrong_number:</strong> guess for did we have the wrong number for this person (to be reviewed)</li>
+                    <li><strong>names_extract:</strong> guess for what names (if any) were provided by this person as tripling targets (to be reviewed)</li>
+                </ul>
+
+                <li>A file of opt-outs, with the following fields:
+                <ul>
+                    <li><strong>conversationid:</strong> a unique identifier for the conversation</li>
+                    <li><strong>contact_phone:</strong> the phone number of the target</li>
+                    <li><strong>opted_out:</strong> yes/no</li>
+                    <li><strong>wrong_number:</strong> yes/no</li>
+            </ul>
+        </div>
+
+
         <div class="upload-enabled">
             <hr />
 
@@ -168,100 +269,6 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
 
         </div>
 
-        <div class="upload-enabled">
-            <hr />
-
-            <h2 id="sms_agg">SMS Aggregation</h2>
-            <blockquote>I need to aggregate SMS messages by conversation. This step is necessary before performing any extraction on SMS data.</blockquote>
-
-            <p>Upload a csv file of raw individual SMS messages. The csv should contain the following columns:</p>
-            <ul>
-                <li><strong>MessageId:</strong> Message timestamp OR an ascending unique ID for the message, used to determine message order</li>
-                <li><strong>ConversationId:</strong> unique ID for each conversation</li>
-                <li><strong>MessageDirection:</strong> direction ("inbound" or "outbound") of each message</li>
-                <li><strong>MessageBody:</strong> the body/content of the message</li>
-                <li><strong>EndpointPhoneNumber:</strong> phone number or any other unique identifier for the recipient</li>
-            </ul>
-
-            <form method="post" enctype="multipart/form-data">
-                <div>
-                    <label for="smsagg_email">Your email:</label>
-                    <input type="email" id="smsagg_email" name="smsagg_email" size="30"/>
-                </div>
-                <div>
-                    <label for="smsagg_file">Your CSV:</label>
-                    <input type="file" id="smsagg_file" name="smsagg_file" accept=".csv" />
-                    <input type="submit" value="Upload" />
-                </div>
-            </form>
-
-            <p id="sms_agg_output">If successful, the output will be a CSV with a single row representing each text message conversation, including the following fields:
-            <ul>
-                <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
-                <li><strong>contact_phone:</strong> the phone number of the target</li>
-                <li><strong>totalMessages:</strong> the total number of messages exchanged</li>
-                <li><strong>tripleMessage:</strong> initial message sent from the text banker to the target</li>
-                <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
-                <li><strong>tripleResponse:</strong> follow up message sent from the text banker to the target</li>
-                <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
-                <li><strong>tripleFinal:</strong> final follow up sent by text banker</li>
-                <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
-                <li><strong>noResponse:</strong> boolean for whether there was no response</li>
-                <li><strong>negResponse:</strong> boolean for generally negative or discouraging terms (sorry, no, etc.)</li>
-                <li><strong>posResponse:</strong> boolean for generally positive or encouraging terms</li>
-                <li><strong>affirmResponse:</strong> boolean for presence of a scripted affirmation by text banker</li>
-                <li><strong>finalAffirmResponse:</strong> boolean for presence of a scripted follow up affirmation by text banker</li>
-            </ul>
-
-        </div>
-
-        <div class="upload-enabled">
-            <hr />
-            <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Extraction</h2>
-            <blockquote>I have SMS conversations and I need to figure out which text recipients volunteered to triple, which chose to opt out, what names they provided, and whether they moved.</blockquote>
-
-            <p>Upload a csv file of the same format as the <a href="#sms_agg_output">output of SMS Aggregation</a></p>
-            
-            <form method="post" enctype="multipart/form-data">
-                <div>
-                    <label for="sccne_email">Your email:</label>
-                    <input type="email" id="sccne_email" name="sccne_email" size="30"/>
-                </div>
-                <div>
-                    <label for="sccne_file">Your CSV:</label>
-                    <input type=file id="sccne_file" name="sccne_file" accept=".csv">
-                    <input type="submit" value="Upload"/>
-                </div>
-            </form>
-
-            <p>If successful, the output will include two files:
-            <ul>
-                <li>A file of triplers. For each tripler, we provide the following fields (each row represents one text message conversation):
-                <ul>
-                    <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
-                    <li><strong>contact_phone:</strong> the phone number of the target</li>
-                    <li><strong>is_tripler:</strong> did this person agree to be a tripler ('yes' for everyone in this file)</li>
-                    <li><strong>opted_out:</strong> did this person opt out of future messages</li>
-                    <li><strong>wrong_number:</strong> did we have the wrong number for this person</li>
-                    <li><strong>names_extract:</strong> what names (if any) were provided by this person as tripling targets</li>
-                </ul>
-
-                <li>A file of conversations for manual review, with the following fields:
-                <ul>
-                    <li><strong>ConversationId:</strong> a unique identifier for the conversation</li>
-                    <li><strong>contact_phone:</strong> the phone number of the target</li>
-                    <li><strong>voterResponse:</strong> initial response(s) by the target (generally where the target makes known if they opt out or want to triple)</li>
-                    <li><strong>voterFinal:</strong> the final follow up message sent by the target (generally where they provide names)</li>
-                    <li><strong>voterPost:</strong> post script from the target (generally a thank you or good luck)</li>
-                    <li><strong>is_tripler:</strong> guess for did this person agree to be a tripler (to be reviewed)</li>
-                    <li><strong>opted_out:</strong> guess for did this person opt out of future messages (to be reviewed)</li>
-                    <li><strong>wrong_number:</strong> guess for did we have the wrong number for this person (to be reviewed)</li>
-                    <li><strong>names_extract:</strong> guess for what names (if any) were provided by this person as tripling targets (to be reviewed)</li>
-                </ul>
-            </ul>
-        </div>
-
-
     </main>
 
     <script type="text/javascript">

From 066347a1c36efc275e6526a65d1ab1fc22aeb0fe Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 28 Apr 2021 16:21:22 -0600
Subject: [PATCH 41/52] support more args for aggregation

---
 parser/main.py                    | 113 +++++++++++++++---------------
 parser/schema.sql                 |   3 +-
 parser/templates/upload_form.html |  24 +++++--
 3 files changed, 79 insertions(+), 61 deletions(-)

diff --git a/parser/main.py b/parser/main.py
index adfd172..f4691fe 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -55,6 +55,7 @@ def allowed_file(file):
         and file.filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
         and file.content_type == 'text/csv')
 
+
 def unique_filename():
     return secure_filename(
         '{}-{}.csv'.format(
@@ -63,6 +64,7 @@ def unique_filename():
         )
     )
 
+
 def check_headers(file, upload_type):
     headers = file.readline().decode('utf-8').rstrip().split(',')
     for header in UPLOAD_TYPES[upload_type]['required_headers']:
@@ -71,9 +73,9 @@ def check_headers(file, upload_type):
     return True
 
 
-def queue_job(file_path, job_type, email):
-    query = 'INSERT INTO jobs (input_file, job_type, email, status) VALUES (?, ?, ?, ?);'
-    db.query_db(query, (file_path, job_type, email, 'queued'), write=True)
+def queue_job(file_path, job_type, email, other_input=None):
+    query = 'INSERT INTO jobs (input_file, job_type, email, status, other_input) VALUES (?, ?, ?, ?, ?);'
+    db.query_db(query, (file_path, job_type, email, 'queued', other_input), write=True)
     return True
 
 
@@ -81,17 +83,17 @@ def process_job():
     """
         TODO: Replace with a celery queue and workers.
     """
-
-    # if there are any queued jobs, process the oldest one
-    query = 'SELECT id, input_file, job_type FROM jobs WHERE status = ? ORDER BY created_at ASC;'
-    job = db.query_db(query, ('queued',), one=True)
-    if not job:
-        print("No jobs to process")
-        return False
-    job_id = job['id']
-    input_file = job['input_file']
-    job_type = job['job_type']
-    print("Processing job {} file {} type {}".format(job_id, input_file, job_type))
+    if not current_app.config['PROCESS_ASYNC']:
+        # if there are any queued jobs, process the oldest one
+        query = 'SELECT id, input_file, job_type, other_input FROM jobs WHERE status = ? ORDER BY created_at ASC;'
+        job = db.query_db(query, ('queued',), one=True)
+        if not job:
+            print("No jobs to process")
+            return False
+        job_id = job['id']
+        input_file = job['input_file']
+        job_type = job['job_type']
+        print("Processing job {} file {} type {}".format(job_id, input_file, job_type))
     
     status = 'success'
     processing_query = 'UPDATE jobs SET status = ? WHERE id = ?;'
@@ -101,7 +103,6 @@ def process_job():
     # but we can easily change that later.
     scripts_folder = current_app.config['SCRIPTS_FOLDER']
     scripts_home_dir = os.path.dirname(scripts_folder)
-    print('shd', scripts_home_dir)
     output_file = unique_filename()
     second_output_file = None
     third_output_file = None
@@ -109,13 +110,17 @@ def process_job():
         second_output_file = unique_filename()
         if job_type == 'sccne_file':
             third_output_file = unique_filename()
+    if job_type == 'smsagg_file':
+        aff_regex, aff_regex_final, init_triple_phrase = job['other_input'].split('**')
+
     cmd = None
     if job_type == 'tblc_file':
         cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
     elif job_type == 'vec_file':
         cmd = 'python {}/van_export_cleaning.py -i {} -f {} -o {} -m {}'.format(
-            scripts_folder, input_file, scripts_home_dir, output_file, second_output_file)
+            scripts_folder, input_file, scripts_home_dir, output_file,
+            second_output_file)
     elif job_type == 'tblctmc_file':
         cmd = 'python {}/name_cleaning_with_responses.py -i {} -f {} -o {}'.format(
             scripts_folder, input_file, scripts_home_dir, output_file)
@@ -124,11 +129,13 @@ def process_job():
             scripts_folder, input_file, scripts_home_dir, output_file, second_output_file,
             third_output_file)
     elif job_type == 'smsagg_file':
-        cmd = 'python {}/aggregate_text_messages.py -d {} -o {}/{}'.format(
-            scripts_folder, input_file, current_app.config['RESULTS_FOLDER'], output_file)
+        cmd = 'python {}/aggregate_text_messages.py -d {} -o {}/{} -a "{}" -af "{}" -t "{}"'.format(
+            scripts_folder, input_file, current_app.config['RESULTS_FOLDER'],
+            output_file, aff_regex, aff_regex_final, init_triple_phrase)
     else:
         err_log = "Unknown job type {}".format(job_type)
         return False, err_log, None
+    
     print("cmd", cmd)
     job_run = subprocess.run(cmd, stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT, shell=True)
@@ -139,15 +146,13 @@ def process_job():
         print(err_log)
     else:
         print("SUCCESS")
+    
     result_file = None
     if status == 'success':
-        result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], output_file)
-        if second_output_file:
-            result_file = '{0}/{1}|{0}/{2}'.format(
-                current_app.config['RESULTS_FOLDER'], output_file, second_output_file)
-        if third_output_file:
-            result_file = '{0}/{1}|{0}/{2}|{0}/{3}'.format(
-                current_app.config['RESULTS_FOLDER'], output_file, second_output_file, third_output_file)
+        result_files = list(filter(
+            None, [output_file, second_output_file, third_output_file]))
+        result_file = '|'.join([
+            '{}/{}'.format(current_app.config['RESULTS_FOLDER'], file) for file in result_files])
     done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
     update = db.query_db(done_query, (status, result_file, job_id), write=True)
     if status == 'success':
@@ -198,43 +203,41 @@ def index():
             file.save(file_path)
             email_name = '{}_email'.format(upload_type.split('_')[0])
             email = request.form[email_name]
-            result = queue_job(file_path, upload_type, email)
-
-            # This message is currently a lie, we're not really queueing jobs now
-            # except for very briefly.
-            # queue_msg = ('Queued file {} for processing as {}. Check your email {} '
-            #        'in a few minutes for results.').format(
-            #            filename,
-            #            UPLOAD_TYPES[upload_type]['name'],
-            #            email)
-            # queue_msg = ('Processing file {} as {}'.format(
-            #                 filename, UPLOAD_TYPES[upload_type]['name']))
-            # flash(queue_msg, 'info')
-            success, output, second_output, third_output = process_job()
-            output_files = [output, second_output, third_output]
-            if success:
-                outcome_msg = Markup(
-                    '<a href="/results/{}">Download results</a>'.format(output))
-                if second_output:
-                    result_links = ['<a href="/results/{}">file {}</a>'.format(
-                        x, output_files.index(x) + 1) for x in output_files]
+            aff_regex = None
+            aff_regex_final = None
+            init_triple_phrase = None
+            if upload_type == 'smsagg_file':
+                aff_regex = request.form['aff_regex']
+                aff_regex_final = request.form['aff_regex_final']
+                init_triple_phrase = request.form['init_triple_phrase']
+
+            outcome_msg = ('Queued file {} for processing as {}. Check your email {} '
+                   'in a few minutes for results.').format(
+                       filename, UPLOAD_TYPES[upload_type]['name'], email)
+
+            if not current_app.config['PROCESS_ASYNC']:
+                result = queue_job(file_path, upload_type, email, '{}**{}**{}'.format(aff_regex, aff_regex_final, init_triple_phrase))
+                success, output, second_output, third_output = process_job()
+                output_files = [output, second_output, third_output]
+                if success:
                     outcome_msg = Markup(
-                        'Download results:  {}'.format('  '.join(result_links)))
-            else:
-                outcome_msg = 'Error processing file {}'.format(
-                    output) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
+                        '<a href="/results/{}">Download results</a>'.format(output))
+                    if second_output:
+                        result_links = ['<a href="/results/{}">file {}</a>'.format(
+                            x, output_files.index(x) + 1) for x in output_files]
+                        outcome_msg = Markup(
+                            'Download results:  {}'.format('  '.join(result_links)))
+                else:
+                    outcome_msg = 'Error processing file {}'.format(
+                        output) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
+
             flash(outcome_msg, 'info')
             return redirect(request.url)
     # GET
     return render_template('upload_form.html')
 
 
-# returning results files
 @bp.route('/results/<filename>')
-def uploaded_file(filename):
+def results_file(filename):
     return send_from_directory(current_app.config['RESULTS_FOLDER'],
                                filename)
-    # TODO: documentation on setting file directory permissions
-    # i.e. results dir can't be inspected
-
-
diff --git a/parser/schema.sql b/parser/schema.sql
index 3f8854b..a9ace5c 100644
--- a/parser/schema.sql
+++ b/parser/schema.sql
@@ -7,5 +7,6 @@ CREATE TABLE jobs (
   result_file TEXT,
   job_type TEXT NOT NULL,
   email TEXT NOT NULL,
-  status TEXT NOT NULL
+  status TEXT NOT NULL,
+  other_input TEXT
 );
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 004eb45..82b99e3 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -85,12 +85,26 @@ <h2 id="sms_agg">SMS Aggregation</h2>
 
             <form method="post" enctype="multipart/form-data">
                 <div>
-                    <label for="smsagg_email">Your email:</label>
+                    <label for="smsagg_email">Your email (to receive results):</label>
                     <input type="email" id="smsagg_email" name="smsagg_email" size="30"/>
                 </div>
                 <div>
                     <label for="smsagg_file">Your CSV:</label>
                     <input type="file" id="smsagg_file" name="smsagg_file" accept=".csv" />
+                </div>
+                <div>
+                    <label for="aff_regex">(optional) Regular expression to detect phrases used by text bankers to affirm that the voter has responded that they are willing to participate. Default value is <strong>what are the [a-z]+ names</strong>.</label>
+                    <div><input type="text" id="aff_regex" name="aff_regex" value="what are the [a-z]+ names" size="50"/></div>
+                </div>
+                <div>
+                    <label for="aff_regex_final">(optional) Regular expression used to detect phrases used by text bankers to affirm that the voter has provided names. Default value is <strong>ways to vote|how to vote|a reminder to vote can make all the difference</strong>.</label>
+                    <div><input type="text" id="aff_regex_final" name="aff_regex_final" value="ways to vote|how to vote|a reminder to vote can make all the difference" size="60"/></div>
+                </div>
+                <div>
+                    <label for="init_triple_phrase">(optional) Regular expression used to mark the initial intro phrase used by text bankers. We will eliminate any messages before this phrase as anomalous. Leave blank if you don't want to remove anything. Default value is (blank).</label>
+                    <div><input type="text" id="init_triple_phrase" name="init_triple_phrase" value="" size="60"/></div>
+                </div>
+                <div>
                     <input type="submit" value="Upload" />
                 </div>
             </form>
@@ -124,7 +138,7 @@ <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Ext
             
             <form method="post" enctype="multipart/form-data">
                 <div>
-                    <label for="sccne_email">Your email:</label>
+                    <label for="sccne_email">Your email (to receive results):</label>
                     <input type="email" id="sccne_email" name="sccne_email" size="30"/>
                 </div>
                 <div>
@@ -181,7 +195,7 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
 
             <form method="post" enctype="multipart/form-data">
                 <div>
-                    <label for="tblc_email">Your email:</label>
+                    <label for="tblc_email">Your email (to receive results):</label>
                     <input type="email" id="tblc_email" name="tblc_email" size="30"/>
                 </div>
                 <div>
@@ -208,7 +222,7 @@ <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (ut
 
             <form method="post" enctype="multipart/form-data">
                 <div>
-                    <label for="tblctmc_email">Your email:</label>
+                    <label for="tblctmc_email">Your email (to receive results):</label>
                     <input type="email" id="tblctmc_email" name="tblctmc_email" size="30"/>
                 </div>
                 <div>
@@ -239,7 +253,7 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
 
             <form method="post" enctype="multipart/form-data">
                 <div>
-                    <label for="vec_email">Your email:</label>
+                    <label for="vec_email">Your email (to receive results):</label>
                     <input type="email" id="vec_email" name="vec_email" size="30"/>
                 </div>
                 <div>

From 183616cbdb2867db1b9a7985f1c4776fe7a57e30 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Wed, 28 Apr 2021 17:15:15 -0600
Subject: [PATCH 42/52] removed db to simplify and all works in synchronous
 mode

---
 README_local.md    |  13 ++--
 parser/__init__.py |   8 +--
 parser/db.py       |  54 --------------
 parser/main.py     | 175 ++++++++++++++++++++++++++++-----------------
 4 files changed, 115 insertions(+), 135 deletions(-)
 delete mode 100644 parser/db.py

diff --git a/README_local.md b/README_local.md
index 501d53a..0e2712f 100644
--- a/README_local.md
+++ b/README_local.md
@@ -131,7 +131,7 @@ This script will output two files:
 - *names_extract* a guess for the extracted names (to be reviewed)
 
 # Running the app frontend
-app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and requesting that the above scripts be run with them. If config.PROCESS_ASYNC is set to True, it uses Celery workers, a Redis queue and Flask-mail to manage script jobs and email notifications with the results in the background. If config.PROCESS_ASYNC is set to False, it runs script jobs synchronously from a queue/log stored in a SQLite database.
+app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and requesting that the above scripts be run with them. If config.PROCESS_ASYNC is set to True, it uses Celery workers, a Redis queue and Flask-mail to manage script jobs and email notifications in the background. If config.PROCESS_ASYNC is set to False, it runs script jobs synchronously and the app waits to deliver the results as linked files.
 
 Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
 
@@ -139,15 +139,12 @@ You'll need to install Redis. On OSX, install homebrew and then `brew install re
 
 Copy config.py from `instance/config.py.example` file and fill it in.
 
-To run an instance of the frontend locally, from the project root directory initialize the db:
-`
+To run an instance of the frontend locally, from the project root directory run:
+```
 export FLASK_APP=parser
 export FLASK_ENV=development
-flask init-db
-`
-That will create a file named `parser.sqlite3` (the application database) in the `instance` directory. Then you should be able to do
-`flask run
-`
+flask run
+```
 and access the running application at [http://localhost:5000/](http://localhost:5000/)
 
 ## Configuring email
diff --git a/parser/__init__.py b/parser/__init__.py
index ea89b30..b9902c6 100644
--- a/parser/__init__.py
+++ b/parser/__init__.py
@@ -28,15 +28,9 @@ def create_app(test_config=None):
         # load the test config if passed in
         app.config.from_mapping(test_config)
 
-    import db
-    db.init_app(app)
-
     import main
     app.register_blueprint(main.bp)
 
-    import tasks
-    app.register_blueprint(tasks.bp)
-
     # ensure the instance folder exists
     try:
         os.makedirs(app.instance_path)
@@ -46,4 +40,4 @@ def create_app(test_config=None):
     # redis_client.init_app(app)
     celery.conf.update(app.config)
 
-    return app
\ No newline at end of file
+    return app
diff --git a/parser/db.py b/parser/db.py
deleted file mode 100644
index 8d4df7a..0000000
--- a/parser/db.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import sqlite3
-
-import click
-from flask import current_app, g
-from flask.cli import with_appcontext
-
-
-def get_db():
-    if 'db' not in g:
-        g.db = sqlite3.connect(
-            current_app.config['DATABASE'],
-            detect_types=sqlite3.PARSE_DECLTYPES
-        )
-        g.db.row_factory = sqlite3.Row
-
-    return g.db
-
-def query_db(query, args=(), one=False, write=False):
-    if write:
-        conn = get_db()
-        conn.execute(query, args)
-        conn.commit()
-        return True
-    cur = get_db().execute(query, args)
-    rv = cur.fetchall()
-    cur.close()
-    return (rv[0] if rv else None) if one else rv
-
-
-def close_db(e=None):
-    db = g.pop('db', None)
-
-    if db is not None:
-        db.close()
-
-
-def init_db():
-    db = get_db()
-
-    with current_app.open_resource('schema.sql') as f:
-        db.executescript(f.read().decode('utf8'))
-
-
-@click.command('init-db')
-@with_appcontext
-def init_db_command():
-    """Clear the existing data and create new tables."""
-    init_db()
-    click.echo('Initialized the database.')
-
-
-def init_app(app):
-    app.teardown_appcontext(close_db)
-    app.cli.add_command(init_db_command)
\ No newline at end of file
diff --git a/parser/main.py b/parser/main.py
index f4691fe..f6b06e3 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -3,12 +3,10 @@
 import functools
 import os
 import random
-import sqlite3
 import string
 import subprocess
 
 import config
-import db
 
 from __init__ import celery
 from flask import (Blueprint, current_app, flash, Flask, g, Markup,
@@ -73,36 +71,14 @@ def check_headers(file, upload_type):
     return True
 
 
-def queue_job(file_path, job_type, email, other_input=None):
-    query = 'INSERT INTO jobs (input_file, job_type, email, status, other_input) VALUES (?, ?, ?, ?, ?);'
-    db.query_db(query, (file_path, job_type, email, 'queued', other_input), write=True)
-    return True
-
-
-def process_job():
-    """
-        TODO: Replace with a celery queue and workers.
-    """
-    if not current_app.config['PROCESS_ASYNC']:
-        # if there are any queued jobs, process the oldest one
-        query = 'SELECT id, input_file, job_type, other_input FROM jobs WHERE status = ? ORDER BY created_at ASC;'
-        job = db.query_db(query, ('queued',), one=True)
-        if not job:
-            print("No jobs to process")
-            return False
-        job_id = job['id']
-        input_file = job['input_file']
-        job_type = job['job_type']
-        print("Processing job {} file {} type {}".format(job_id, input_file, job_type))
-    
-    status = 'success'
-    processing_query = 'UPDATE jobs SET status = ? WHERE id = ?;'
-    update = db.query_db(processing_query, ('processing', job_id), write=True)
-
+@celery.task
+def process_job(data):
     # It seems silly to run these via the command line
     # but we can easily change that later.
-    scripts_folder = current_app.config['SCRIPTS_FOLDER']
+    scripts_folder = config.SCRIPTS_FOLDER
     scripts_home_dir = os.path.dirname(scripts_folder)
+    input_file = data['input_file']
+    job_type = data['upload_type']
     output_file = unique_filename()
     second_output_file = None
     third_output_file = None
@@ -110,8 +86,6 @@ def process_job():
         second_output_file = unique_filename()
         if job_type == 'sccne_file':
             third_output_file = unique_filename()
-    if job_type == 'smsagg_file':
-        aff_regex, aff_regex_final, init_triple_phrase = job['other_input'].split('**')
 
     cmd = None
     if job_type == 'tblc_file':
@@ -130,49 +104,74 @@ def process_job():
             third_output_file)
     elif job_type == 'smsagg_file':
         cmd = 'python {}/aggregate_text_messages.py -d {} -o {}/{} -a "{}" -af "{}" -t "{}"'.format(
-            scripts_folder, input_file, current_app.config['RESULTS_FOLDER'],
-            output_file, aff_regex, aff_regex_final, init_triple_phrase)
+            scripts_folder, input_file, config.RESULTS_FOLDER,
+            output_file, data['aff_regex'], data['aff_regex_final'],
+            data['init_triple_phrase'])
     else:
         err_log = "Unknown job type {}".format(job_type)
-        return False, err_log, None
-    
+        return False, err_log, None, None
+
+    status = 'success'
     print("cmd", cmd)
     job_run = subprocess.run(cmd, stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT, shell=True)
     if job_run.returncode: # exit code of 0 is success, 1 is generic error
         status = 'error'
         err_log = job_run.stdout.decode()
-        print("ERROR: Could not process job {} file {} type {}".format(job_id, input_file, job_type))
+        print("ERROR: Could not process job {} file {} type {}".format(
+            job_id, input_file, job_type))
         print(err_log)
     else:
         print("SUCCESS")
     
-    result_file = None
-    if status == 'success':
-        result_files = list(filter(
-            None, [output_file, second_output_file, third_output_file]))
-        result_file = '|'.join([
-            '{}/{}'.format(current_app.config['RESULTS_FOLDER'], file) for file in result_files])
-    done_query = 'UPDATE jobs SET status = ?, result_file = ? WHERE id = ?;'
-    update = db.query_db(done_query, (status, result_file, job_id), write=True)
     if status == 'success':
+        if config.PROCESS_ASYNC:
+            result_files = list(filter(
+                None, [output_file, second_output_file, third_output_file]))
+            result_file = '|'.join([
+                '{}/{}'.format(config.RESULTS_FOLDER, file) for file in result_files])
+            email_data = {
+                'result_file': result_file,
+                'email': data['email'] or config.TEST_TARGET_EMAIL,
+                'upload_type': upload_type
+            }
+            res = email_results.apply_async(args=[email_data], countdown=0)
+        # TODO: delete input file. If processing wasn't successful, leave the input file
+        # for troubleshooting
         return True, output_file, second_output_file, third_output_file
-    return False, err_log, None, None
+    else:
+        return False, err_log, None, None
 
 
-def email_results():
-    pass
+@celery.task
+def email_results(data):
+    with current_app.app_context():
+        # for some reason celery spins up a new instance without access to
+        # original config file, so we re-config here
+        current_app.config.from_object(config)
+        mail = Mail(current_app)
+        msg = Message(config.EMAIL_SUBJECT,
+                sender=config.EMAIL_SENDER,
+                recipients=[data['email']])
+        # accommodating more than one output file
+        output = data['result_file'].split('|')
+        results = ''.join(['<p>{}results/{}</p>'.format(config.BASE_URL, output_file) for output_file in output])
+        msg.html = (
+            "<p>Thank you for using the votetripling.org SMS transcript processing tool.</p>"
+            "<p>The {} script successfully processed your data file.</p>"
+            "<p>Link(s) to download the results:</p>"
+            "{}"
+            "<p>Your result files will be available for download for {} hours.</p>"
+        ).format(UPLOAD_TYPES[data['upload_type']]['name'], results, config.FILE_LIFE)
+        mail.send(msg)
+    return True
 
 
-def cleanup_files(filename):
-    # delete files that are more than 48 hours old
+def cleanup_files(interval=72):
+    # delete output files that are more than 72 hours old
     pass
 
 
-def cleanup_jobs(interval):
-    # delete jobs that are more than `interval` old, to keep the db from getting too big
-    pass
-
 # most of the business is here
 @bp.route('/', methods=['GET', 'POST'])
 def index():
@@ -211,22 +210,30 @@ def index():
                 aff_regex_final = request.form['aff_regex_final']
                 init_triple_phrase = request.form['init_triple_phrase']
 
-            outcome_msg = ('Queued file {} for processing as {}. Check your email {} '
+            outcome_msg = ('Queued file {} for processing as {}. Check your email '
                    'in a few minutes for results.').format(
-                       filename, UPLOAD_TYPES[upload_type]['name'], email)
-
-            if not current_app.config['PROCESS_ASYNC']:
-                result = queue_job(file_path, upload_type, email, '{}**{}**{}'.format(aff_regex, aff_regex_final, init_triple_phrase))
-                success, output, second_output, third_output = process_job()
-                output_files = [output, second_output, third_output]
+                       filename, UPLOAD_TYPES[upload_type]['name'])
+            data = {
+                'input_file': file_path,
+                'upload_type': upload_type,
+                'email': email,
+                'aff_regex': aff_regex,
+                'aff_regex_final': aff_regex_final,
+                'init_triple_phrase': init_triple_phrase
+            }
+            
+            if current_app.config['PROCESS_ASYNC']:
+                res = process_job.apply_async(args=[data], countdown=0)
+            else:
+                # make the user wait for processing
+                print("Processing file {} type {}".format(file_path, upload_type))
+                success, output, second_output, third_output = process_job(data)
                 if success:
+                    output_files = list(filter(None, [output, second_output, third_output]))
+                    result_links = ['<a href="/results/{}" style="padding: 0 1em;">File {}</a>'.format(
+                        x, output_files.index(x) + 1) for x in output_files]
                     outcome_msg = Markup(
-                        '<a href="/results/{}">Download results</a>'.format(output))
-                    if second_output:
-                        result_links = ['<a href="/results/{}">file {}</a>'.format(
-                            x, output_files.index(x) + 1) for x in output_files]
-                        outcome_msg = Markup(
-                            'Download results:  {}'.format('  '.join(result_links)))
+                        'Download results:  {}'.format('  '.join(result_links)))
                 else:
                     outcome_msg = 'Error processing file {}'.format(
                         output) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
@@ -241,3 +248,39 @@ def index():
 def results_file(filename):
     return send_from_directory(current_app.config['RESULTS_FOLDER'],
                                filename)
+
+
+# for testing only
+@bp.route('/email-results/', methods=['GET'])
+def email():
+    if request.method == 'GET':
+        data = {
+            'job_id': 1234,
+            'upload_type': 'vec_file',
+            'result_file': '123.csv|456.csv',
+            'email': current_app.config['TEST_TARGET_EMAIL']
+        }
+        res = email_results.apply_async(args=[data], countdown=0)
+        if res.failed():
+            print('emailing results failed')
+        msg = ('Email queued for send! Check your inbox (and spam) for an email '
+            'from "{}" with the subject "{}".').format(
+                current_app.config['EMAIL_SENDER'], current_app.config['EMAIL_SUBJECT'])
+        return msg
+    return 'not allowed here'
+
+
+# to manually kick off processing
+@bp.route('/process/<upload_type>/<file_name>', methods=['GET'])
+def process():
+    if request.method == 'GET':
+        data = {
+            'input_file': '{}/{}'.format(current_app.config['UPLOAD_FOLDER'], file_name),
+            'upload_type': upload_type,
+            'email': None
+        }
+        res = process_job_async.apply_async(args=[data], countdown=0)
+        msg = ('File {} queued for processing as {}. Results will be emailed to {}.').format(
+            file_name, upload_type, current_app.config['TEST_TARGET_EMAIL'])
+        return msg
+    return 'not allowed here'

From 4d295ce59258f0169cc43c85f80875c8bd115bc4 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 29 Apr 2021 09:50:23 -0600
Subject: [PATCH 43/52] works in async mode

---
 parser/main.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/parser/main.py b/parser/main.py
index f6b06e3..8379339 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -123,17 +123,16 @@ def process_job(data):
         print(err_log)
     else:
         print("SUCCESS")
-    
+
     if status == 'success':
         if config.PROCESS_ASYNC:
             result_files = list(filter(
                 None, [output_file, second_output_file, third_output_file]))
-            result_file = '|'.join([
-                '{}/{}'.format(config.RESULTS_FOLDER, file) for file in result_files])
+            result_file = '|'.join([file for file in result_files])
             email_data = {
                 'result_file': result_file,
                 'email': data['email'] or config.TEST_TARGET_EMAIL,
-                'upload_type': upload_type
+                'job_type': job_type
             }
             res = email_results.apply_async(args=[email_data], countdown=0)
         # TODO: delete input file. If processing wasn't successful, leave the input file
@@ -162,7 +161,7 @@ def email_results(data):
             "<p>Link(s) to download the results:</p>"
             "{}"
             "<p>Your result files will be available for download for {} hours.</p>"
-        ).format(UPLOAD_TYPES[data['upload_type']]['name'], results, config.FILE_LIFE)
+        ).format(UPLOAD_TYPES[data['job_type']]['name'], results, config.FILE_LIFE)
         mail.send(msg)
     return True
 
@@ -212,7 +211,7 @@ def index():
 
             outcome_msg = ('Queued file {} for processing as {}. Check your email '
                    'in a few minutes for results.').format(
-                       filename, UPLOAD_TYPES[upload_type]['name'])
+                       file.filename, UPLOAD_TYPES[upload_type]['name'])
             data = {
                 'input_file': file_path,
                 'upload_type': upload_type,

From a34f940ad3df3c704c02f7281832416151f053eb Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Tue, 4 May 2021 11:09:23 -0600
Subject: [PATCH 44/52] file cleanup now works

---
 README_local.md |  19 +++++--
 parser/main.py  |  46 +++++++++++++---
 parser/tasks.py | 139 ------------------------------------------------
 3 files changed, 52 insertions(+), 152 deletions(-)
 delete mode 100644 parser/tasks.py

diff --git a/README_local.md b/README_local.md
index 0e2712f..c0a6d50 100644
--- a/README_local.md
+++ b/README_local.md
@@ -131,11 +131,11 @@ This script will output two files:
 - *names_extract* a guess for the extracted names (to be reviewed)
 
 # Running the app frontend
-app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and requesting that the above scripts be run with them. If config.PROCESS_ASYNC is set to True, it uses Celery workers, a Redis queue and Flask-mail to manage script jobs and email notifications in the background. If config.PROCESS_ASYNC is set to False, it runs script jobs synchronously and the app waits to deliver the results as linked files.
+app.py is a Python 3.x, Flask-based frontend that provides a dedicated UI for uploading data sets and requesting that the above scripts be run with them. 
 
 Make sure you've created and activated a virtual environment (see Requirements) and installed everything in requirements.txt.
 
-You'll need to install Redis. On OSX, install homebrew and then `brew install redis`. You may also need to run `pip install "celery[redis]"`
+You'll need to [install Redis](https://redis.io/topics/quickstart). On OSX, install homebrew and then `brew install redis`. You may also need to run `pip install "celery[redis]"`
 
 Copy config.py from `instance/config.py.example` file and fill it in.
 
@@ -153,11 +153,20 @@ Email config variables in the example config file assume you are using Gmail for
 * Gmail probably isn't adequate for production scale; you can only send about 100 emails a day.
 * Gmail doesn't consider any apps that send mail using SMTP protocol secure. When you try and run the app with a Gmail account you'll get security warnings on that account unless you have enabled what Google calls ["Less Secure Apps"](https://support.google.com/accounts/answer/6010255?hl=en).
 
-## Background tasks
+## Running scripts async in the background vs. waiting for results
 
-* `celery -A celery_worker.celery worker --loglevel=info` will spin up a celery worker for you in a local dev environment.
+If config.PROCESS_ASYNC is set to True, the app uses Celery workers, a Redis queue and Flask-mail to manage script jobs and email results in the background. If config.PROCESS_ASYNC is set to False, it runs script jobs synchronously and the app waits to deliver the results as linked files. 
+
+Synchronous mode is not recommended for production if you expect lots of large files that take a while ( > 30 seconds) to process.
+
+If you set config.PROCESS_ASYNC to true, you'll need to run celery and redis (which celery uses to manage its queue)
+* `celery -A celery_worker.celery worker --loglevel=info` will spin up a celery worker for you in a local dev environment. [More on celery workers](https://docs.celeryproject.org/en/stable/userguide/workers.html)
 * Run redis in a different terminal window with `redis-server`.
 
 ## Testing the app frontend
 
-`pytest` should run all the tests in the `tests` folder.
\ No newline at end of file
+`pytest` should run all the tests in the `tests` folder.
+
+## TODO
+
+ A Docker container would ease of deployment.
\ No newline at end of file
diff --git a/parser/main.py b/parser/main.py
index 8379339..3cb94f8 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -71,7 +71,7 @@ def check_headers(file, upload_type):
     return True
 
 
-@celery.task
+@celery.task(time_limit=1800)
 def process_job(data):
     # It seems silly to run these via the command line
     # but we can easily change that later.
@@ -135,6 +135,7 @@ def process_job(data):
                 'job_type': job_type
             }
             res = email_results.apply_async(args=[email_data], countdown=0)
+            cleanup = clean_files.apply_async(args=[data], countdown=10)
         # TODO: delete input file. If processing wasn't successful, leave the input file
         # for troubleshooting
         return True, output_file, second_output_file, third_output_file
@@ -165,11 +166,31 @@ def email_results(data):
         mail.send(msg)
     return True
 
-
-def cleanup_files(interval=72):
-    # delete output files that are more than 72 hours old
-    pass
-
+@celery.task
+def cleanup_files(data=None):
+    # delete input file
+    print('FILE CLEANUP')
+    if data:
+        input_file = data['input_file']
+        if os.path.exists(input_file):
+            print('deleting input file {}'.format(input_file))
+            os.remove(input_file)
+        else:
+            print('{} does not exist'.format(input_file))
+    # delete output files that are more than config.FILE_LIFE hours old
+    print('deleting results files')
+    output_folder = config.RESULTS_FOLDER
+    files = os.listdir(config.RESULTS_FOLDER)
+    for file in files:
+        try:
+            date_time_obj = datetime.datetime.strptime(
+                file.split('-')[1].replace('.csv', ''), '%Y%m%d%H%M%S')
+        except:
+            continue
+        if date_time_obj + datetime.timedelta(hours=config.FILE_LIFE) < datetime.datetime.now():
+            print(file)
+            os.remove('{}/{}'.format(config.RESULTS_FOLDER, file))
+    return True
 
 # most of the business is here
 @bp.route('/', methods=['GET', 'POST'])
@@ -249,7 +270,7 @@ def results_file(filename):
                                filename)
 
 
-# for testing only
+# test emailing
 @bp.route('/email-results/', methods=['GET'])
 def email():
     if request.method == 'GET':
@@ -269,7 +290,7 @@ def email():
     return 'not allowed here'
 
 
-# to manually kick off processing
+# manually kick off processing
 @bp.route('/process/<upload_type>/<file_name>', methods=['GET'])
 def process():
     if request.method == 'GET':
@@ -283,3 +304,12 @@ def process():
             file_name, upload_type, current_app.config['TEST_TARGET_EMAIL'])
         return msg
     return 'not allowed here'
+
+
+# manually kick off file cleanup
+@bp.route('/cleanup/', methods=['GET'])
+def cleanup():
+    if request.method == 'GET':
+        res = cleanup_files.apply_async(countdown=0)
+        return ('deleting result files older than {} hours'.format(config.FILE_LIFE))
+    return 'not allowed here'
\ No newline at end of file
diff --git a/parser/tasks.py b/parser/tasks.py
deleted file mode 100644
index ef6793a..0000000
--- a/parser/tasks.py
+++ /dev/null
@@ -1,139 +0,0 @@
-""" 
-    TODO:
-    Production implementation should process jobs and 
-    send emails with a queue and workers.
-"""
-from __init__ import celery
-from flask import Blueprint, current_app, request
-from flask_mail import Mail, Message
-from main import UPLOAD_TYPES, unique_filename
-import config
-
-
-bp = Blueprint('tasks', __name__, url_prefix='/tasks')
-
-@celery.task
-def process_job_async(data):
-    # It seems silly to run these via the command line
-    # but we can easily change this later. Trying to avoid
-    # messing with scripts unless we absolutely have to.
-    input_file = data['input_file']
-    upload_type = data['upload_type']
-    email = data['email']
-    scripts_folder = config.SCRIPTS_FOLDER
-    scripts_home_dir = os.path.dirname(scripts_folder)
-    output_file = unique_filename()
-    second_output_file = None
-    third_output_file = None
-    if job_type in ['vec_file','sccne_file']:
-        second_output_file = unique_filename()
-        if job_type == 'sccne_file':
-            third_output_file = unique_filename()
-
-    if upload_type == 'tblc_file':
-        cmd = 'python {}/name_cleaning.py -i {} -f {} -o {}'.format(
-            scripts_folder, input_file, scripts_home_dir, output_file)
-    elif upload_type == 'vec_file':
-        cmd = 'python {}/van_export_cleaning.py -i {} -f {} -o {} -m {}'.format(
-            scripts_folder, input_file, scripts_home_dir, output_file, second_output_file)
-    elif upload_type == 'tblctmc_file':
-        cmd = 'python {}/name_cleaning_with_responses.py -i {} -f {} -o {}'.format(
-            scripts_folder, input_file, scripts_home_dir, output_file)
-    elif upload_type == 'sccne_file':
-        cmd = 'python {}/annotate_conversations.py -i {} -f {} -o {} -n {} -m {}'.format(
-            scripts_folder, input_file, scripts_home_dir, output_file,
-            second_output_file, third_output_file)
-    elif upload_type == 'smsagg_file':
-        cmd = 'python {}/aggregate_text_messages.py -d {} -o {}'.format(
-            scripts_folder, input_file, output_file)
-    else:
-        print("Unknown job type {}".format(upload_type))
-    print("cmd", cmd)
-    job_run = subprocess.run(cmd, stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT, shell=True)
-    if job_run.returncode: # exit code of 0 is success, 1 is generic error
-        status = 'error'
-        err_log = job_run.stdout.decode()
-        print("ERROR: Could not process job {} file {} type {}".format(job_id, input_file, upload_type))
-        print(err_log)
-    else:
-        print("SUCCESS")
-    result_file = None
-    if status == 'success':
-        result_file = '{}/{}'.format(current_app.config['RESULTS_FOLDER'], output_file)
-        if second_output_file:
-            result_file == '{0}/{1}|{0}/{2}'.format(
-                current_app.config['RESULTS_FOLDER'], output_file, second_output_file)
-    if status == 'success':
-        data = {
-            'result_file': result_file,
-            'email': email or config.TEST_TARGET_EMAIL,
-            'upload_type': upload_type
-        }
-        res = email_results.apply_async(args=[data], countdown=0)
-        # also delete the input file
-        return True, output_file, second_output_file
-    return False, err_log, None
-
-@celery.task
-def email_results(data):
-    with current_app.app_context():
-        # for some reason celery spins up a new instance without access to 
-        # original config file, so we re-config here
-        current_app.config.from_object(config)
-        mail = Mail(current_app)
-        msg = Message(config.EMAIL_SUBJECT,
-                sender=config.EMAIL_SENDER,
-                recipients=[data['email']])
-        # accommodating more than one output file
-        output = data['result_file'].split('|')
-        results = ''.join(['<p>{}results/{}</p>'.format(config.BASE_URL, output_file) for output_file in output])
-        msg.html = (
-            "<p>Thank you for using the votetripling.org SMS transcript processing tool.</p>"
-            "<p>The {} script successfully processed your data file.</p>"
-            "<p>Link(s) to download the results:</p>"
-            "{}"
-            "<p>Your result files will be available for download for {} hours.</p>"
-        ).format(UPLOAD_TYPES[data['upload_type']]['name'], results, config.FILE_LIFE)
-        mail.send(msg)
-    return True
-
-
-# for testing only
-@bp.route('/email-results/', methods=['GET'])
-def email():
-    if request.method == 'GET':
-        data = {
-            'job_id': 1234,
-            'upload_type': 'vec_file',
-            'result_file': '123.csv|456.csv',
-            'email': current_app.config['TEST_TARGET_EMAIL']
-        }
-        res = email_results.apply_async(args=[data], countdown=0)
-        if res.failed():
-            print('emailing results failed')
-        msg = ('Email queued for send! Check your inbox (and spam) for an email '
-            'from "{}" with the subject "{}".').format(
-                current_app.config['EMAIL_SENDER'], current_app.config['EMAIL_SUBJECT'])
-        return msg
-    return 'not allowed here'
-
-# to manually kick off processing - note that this doesn't take an email
-@bp.route('/process/<upload_type>/<file_name>', methods=['GET'])
-def process():
-    if request.method == 'GET':
-        data = {
-            'input_file': '{}/file_name'.format(config.UPLOAD_FOLDER),
-            'upload_type': upload_type,
-            'email': None
-        }
-        res = process_job_async.apply_async(args=[data], countdown=0)
-        msg = ('File {} queued for processing as {}. Results will be emailed to {}.').format(
-            file_name, upload_type, config.TEST_TARGET_EMAIL)
-        return msg
-    return 'not allowed here'
-
-def cleanup_files(filename):
-    # TODO
-    # delete files from finished jobs that are more than n hours old
-    pass

From b680a0fb61c1e2f0fc398a76f852609999225ce0 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Tue, 4 May 2021 12:37:24 -0600
Subject: [PATCH 45/52] no more db

---
 parser/schema.sql | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 parser/schema.sql

diff --git a/parser/schema.sql b/parser/schema.sql
deleted file mode 100644
index a9ace5c..0000000
--- a/parser/schema.sql
+++ /dev/null
@@ -1,12 +0,0 @@
-DROP TABLE IF EXISTS jobs;
-
-CREATE TABLE jobs (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  input_file TEXT NOT NULL,
-  result_file TEXT,
-  job_type TEXT NOT NULL,
-  email TEXT NOT NULL,
-  status TEXT NOT NULL,
-  other_input TEXT
-);

From 23800f704f77c17881fd3a3cd889857a62db9758 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Mon, 17 May 2021 21:17:39 -0600
Subject: [PATCH 46/52] using csv reader allows multiple types of csv
 formatting

---
 parser/main.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/parser/main.py b/parser/main.py
index 3cb94f8..63c49a0 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -63,11 +63,17 @@ def unique_filename():
     )
 
 
-def check_headers(file, upload_type):
-    headers = file.readline().decode('utf-8').rstrip().split(',')
-    for header in UPLOAD_TYPES[upload_type]['required_headers']:
-        if header not in headers:
-            return False
+def check_headers(file_path, upload_type):
+    # open file as csv and read the first line
+    with open(file_path, newline='') as f:
+        reader = csv.reader(f)
+        headers = next(reader)
+        print("headers", headers) # for debugging
+        for header in UPLOAD_TYPES[upload_type]['required_headers']:
+            if header not in headers:
+                f.close()
+                os.remove(file_path)
+                return False
     return True
 
 
@@ -135,7 +141,7 @@ def process_job(data):
                 'job_type': job_type
             }
             res = email_results.apply_async(args=[email_data], countdown=0)
-            cleanup = clean_files.apply_async(args=[data], countdown=10)
+            cleanup = cleanup_files.apply_async(args=[data], countdown=10)
         # TODO: delete input file. If processing wasn't successful, leave the input file
         # for troubleshooting
         return True, output_file, second_output_file, third_output_file
@@ -209,17 +215,16 @@ def index():
             if not (file and allowed_file(file)):
                 flash('Invalid file type. Select a CSV to upload', 'error')
                 return redirect(request.url)
-            if not check_headers(file, upload_type):
+
+            filename = unique_filename()
+            file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
+            file.save(file_path)
+            if not check_headers(file_path, upload_type):
                 msg = 'Invalid CSV headers. Check CSV for required columns {}.'.format(
                     UPLOAD_TYPES[upload_type]['required_headers'])
                 flash(msg, 'error')
                 return redirect(request.url)
 
-            # if it passes all the checks, save file and queue for processing
-            filename = unique_filename()
-            file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename)
-            file.seek(0) # return pointer to beginning of file
-            file.save(file_path)
             email_name = '{}_email'.format(upload_type.split('_')[0])
             email = request.form[email_name]
             aff_regex = None

From 5f879a906349d4ad0b53ec3cdf5dfd6e21217c35 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Mon, 17 May 2021 21:37:54 -0600
Subject: [PATCH 47/52] fixed position messages:

---
 parser/templates/upload_form.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 82b99e3..75912ec 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -14,6 +14,7 @@
         a:hover { text-decoration: underline; color: purple; }
         a:visited { color: rebeccapurple; }
         form div { padding: .3em; margin: .3em .5em; }
+        ul.flashes { position: fixed; top: 0px; left: 50%; width: 500px; margin-left: -300px; opacity: 0.9;}
         ul.flash li { font-size: 1.3em; }
         ul.flashes li.error { list-style-type: none; display: inline-block; background-color: red; padding: 1em; color: #eee; }
         ul.flashes li.info { list-style-type: none; display: inline-block; background-color: darkgreen; padding: 1em; color: #eee; }

From 40db4c7aafa91970ff160818a2033bab71e4e152 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Tue, 18 May 2021 08:31:16 -0600
Subject: [PATCH 48/52] specify inbound and outbound text

---
 parser/main.py                    | 12 +++++++++---
 parser/templates/upload_form.html | 12 ++++++++++--
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/parser/main.py b/parser/main.py
index 63c49a0..9addd52 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -109,10 +109,10 @@ def process_job(data):
             scripts_folder, input_file, scripts_home_dir, output_file, second_output_file,
             third_output_file)
     elif job_type == 'smsagg_file':
-        cmd = 'python {}/aggregate_text_messages.py -d {} -o {}/{} -a "{}" -af "{}" -t "{}"'.format(
+        cmd = 'python {}/aggregate_text_messages.py -d {} -o {}/{} -a "{}" -af "{}" -t "{}" -in "{}"" -out "{}"'.format(
             scripts_folder, input_file, config.RESULTS_FOLDER,
             output_file, data['aff_regex'], data['aff_regex_final'],
-            data['init_triple_phrase'])
+            data['init_triple_phrase'], data['inbound_text'], data['outbound_text'])
     else:
         err_log = "Unknown job type {}".format(job_type)
         return False, err_log, None, None
@@ -230,10 +230,14 @@ def index():
             aff_regex = None
             aff_regex_final = None
             init_triple_phrase = None
+            inbound_text = None
+            outbound_text = None
             if upload_type == 'smsagg_file':
                 aff_regex = request.form['aff_regex']
                 aff_regex_final = request.form['aff_regex_final']
                 init_triple_phrase = request.form['init_triple_phrase']
+                inbound_text = request.form['inbound_text']
+                outbound_text = request.form['outbound_text']
 
             outcome_msg = ('Queued file {} for processing as {}. Check your email '
                    'in a few minutes for results.').format(
@@ -244,7 +248,9 @@ def index():
                 'email': email,
                 'aff_regex': aff_regex,
                 'aff_regex_final': aff_regex_final,
-                'init_triple_phrase': init_triple_phrase
+                'init_triple_phrase': init_triple_phrase, 
+                'inbound_text': inbound_text,
+                'outbound_text': outbound_text
             }
             
             if current_app.config['PROCESS_ASYNC']:
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 75912ec..6d61b0b 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -79,7 +79,7 @@ <h2 id="sms_agg">SMS Aggregation</h2>
             <ul>
                 <li><strong>MessageId:</strong> Message timestamp OR an ascending unique ID for the message, used to determine message order</li>
                 <li><strong>ConversationId:</strong> unique ID for each conversation</li>
-                <li><strong>MessageDirection:</strong> direction ("inbound" or "outbound") of each message</li>
+                <li><strong>MessageDirection:</strong> direction (default: "Inbound" or "Outbound") of each message</li>
                 <li><strong>MessageBody:</strong> the body/content of the message</li>
                 <li><strong>EndpointPhoneNumber:</strong> phone number or any other unique identifier for the recipient</li>
             </ul>
@@ -105,6 +105,14 @@ <h2 id="sms_agg">SMS Aggregation</h2>
                     <label for="init_triple_phrase">(optional) Regular expression used to mark the initial intro phrase used by text bankers. We will eliminate any messages before this phrase as anomalous. Leave blank if you don't want to remove anything. Default value is (blank).</label>
                     <div><input type="text" id="init_triple_phrase" name="init_triple_phrase" value="" size="60"/></div>
                 </div>
+                <div>
+                    <label for="inbound_text">(optional) Exact text of the label given to inbound message in the input data MessageDirection column. Default value is <strong>Inbound</strong>.</label>
+                    <div><input type="text" id="inbound_text" name="inbound_text" value="Inbound" size="60"/></div>
+                </div>
+                <div>
+                    <label for="outbound_text">(optional) Exact text of the label given to outbound message in the input data MessageDirection column. Default value is <strong>Outbound</strong>.</label>
+                    <div><input type="text" id="outbound_text" name="outbound_text" value="Outbound" size="60"/></div>
+                </div>
                 <div>
                     <input type="submit" value="Upload" />
                 </div>
@@ -149,7 +157,7 @@ <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Ext
                 </div>
             </form>
 
-            <p>If successful, the output will include two files:
+            <p>If successful, the output will include three files:
             <ul>
                 <li>A file of triplers. For each tripler, we provide the following fields (each row represents one text message conversation):
                 <ul>

From 47f40fc034df37235fc3b9af4829bb7c999049b5 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 20 May 2021 20:31:25 -0600
Subject: [PATCH 49/52] made file upload and email required in each form

---
 parser/templates/upload_form.html | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 6d61b0b..6c04818 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -87,11 +87,11 @@ <h2 id="sms_agg">SMS Aggregation</h2>
             <form method="post" enctype="multipart/form-data">
                 <div>
                     <label for="smsagg_email">Your email (to receive results):</label>
-                    <input type="email" id="smsagg_email" name="smsagg_email" size="30"/>
+                    <input type="email" id="smsagg_email" name="smsagg_email" size="30" required />
                 </div>
                 <div>
                     <label for="smsagg_file">Your CSV:</label>
-                    <input type="file" id="smsagg_file" name="smsagg_file" accept=".csv" />
+                    <input type="file" id="smsagg_file" name="smsagg_file" accept=".csv" required />
                 </div>
                 <div>
                     <label for="aff_regex">(optional) Regular expression to detect phrases used by text bankers to affirm that the voter has responded that they are willing to participate. Default value is <strong>what are the [a-z]+ names</strong>.</label>
@@ -148,11 +148,11 @@ <h2 id="sms_convo_cat_name_extract">SMS Conversation Categorization and Name Ext
             <form method="post" enctype="multipart/form-data">
                 <div>
                     <label for="sccne_email">Your email (to receive results):</label>
-                    <input type="email" id="sccne_email" name="sccne_email" size="30"/>
+                    <input type="email" id="sccne_email" name="sccne_email" size="30" required />
                 </div>
                 <div>
                     <label for="sccne_file">Your CSV:</label>
-                    <input type=file id="sccne_file" name="sccne_file" accept=".csv">
+                    <input type=file id="sccne_file" name="sccne_file" accept=".csv" required />
                     <input type="submit" value="Upload"/>
                 </div>
             </form>
@@ -205,11 +205,11 @@ <h2 id="text_banker_log_cleaning">Text Banker Log Cleaning</h2>
             <form method="post" enctype="multipart/form-data">
                 <div>
                     <label for="tblc_email">Your email (to receive results):</label>
-                    <input type="email" id="tblc_email" name="tblc_email" size="30"/>
+                    <input type="email" id="tblc_email" name="tblc_email" size="30" required />
                 </div>
                 <div>
                     <label for="tblc_file">Your CSV:</label>
-                    <input type="file" id="tblc_file" name="tblc_file" accept=".csv"/>
+                    <input type="file" id="tblc_file" name="tblc_file" accept=".csv" required/>
                     <input type="submit" value="Upload" />
                 </div>
             </form>
@@ -232,11 +232,11 @@ <h2 id="text_banker_log_cleaning_with_conversation">Text Banker Log Cleaning (ut
             <form method="post" enctype="multipart/form-data">
                 <div>
                     <label for="tblctmc_email">Your email (to receive results):</label>
-                    <input type="email" id="tblctmc_email" name="tblctmc_email" size="30"/>
+                    <input type="email" id="tblctmc_email" name="tblctmc_email" size="30" required/>
                 </div>
                 <div>
                     <label for="tblctmc_file">Your CSV:</label>
-                    <input type="file" id="tblctmc_file" name="tblctmc_file" accept=".csv" />
+                    <input type="file" id="tblctmc_file" name="tblctmc_file" accept=".csv" required />
                     <input type="submit" value="Upload" />
                 </div>
 
@@ -263,11 +263,11 @@ <h2 id="van_export_cleaning">VAN Export Cleaning</h2>
             <form method="post" enctype="multipart/form-data">
                 <div>
                     <label for="vec_email">Your email (to receive results):</label>
-                    <input type="email" id="vec_email" name="vec_email" size="30"/>
+                    <input type="email" id="vec_email" name="vec_email" size="30" required />
                 </div>
                 <div>
                     <label for="vec_file">Your CSV:</label>
-                    <input type="file" id="vec_file" name="vec_file" accept=".csv"/>
+                    <input type="file" id="vec_file" name="vec_file" accept=".csv" required />
                     <input type="submit" value="Upload"/>
                 </div>
             </form>

From 470ed688d97d439ca64f137068d90556bc3e5a99 Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 20 May 2021 20:52:03 -0600
Subject: [PATCH 50/52] add required headers for script 1 and make script 1
 headers match everywhere

---
 Projects/NLP/SMS_Annotation/Code/aggregate_text_messages.py | 2 +-
 parser/main.py                                              | 2 +-
 parser/templates/upload_form.html                           | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Projects/NLP/SMS_Annotation/Code/aggregate_text_messages.py b/Projects/NLP/SMS_Annotation/Code/aggregate_text_messages.py
index 8eaf051..7dcd94d 100644
--- a/Projects/NLP/SMS_Annotation/Code/aggregate_text_messages.py
+++ b/Projects/NLP/SMS_Annotation/Code/aggregate_text_messages.py
@@ -380,7 +380,7 @@ def main(args):
     )
     PARSER.add_argument(
         "-p", "--phoneCol",
-        default="EndpointPhoneNumber",
+        default="ContactPhone",
         help="name of the column in input data containing the phone number. Any unique identifier for the recipient will suffice"
     )
     PARSER.add_argument(
diff --git a/parser/main.py b/parser/main.py
index 9addd52..bd2ee59 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -39,7 +39,7 @@
     },
     'smsagg_file': {
         'name': 'SMS Aggregation',
-        'required_headers': []
+        'required_headers': ['MessageId', 'ConversationId', 'MessageDirection', 'MessageBody', 'ContactPhone']
     },
     'vec_file': {
         'name': 'VAN Export Cleaning',
diff --git a/parser/templates/upload_form.html b/parser/templates/upload_form.html
index 6c04818..3c36757 100644
--- a/parser/templates/upload_form.html
+++ b/parser/templates/upload_form.html
@@ -81,7 +81,7 @@ <h2 id="sms_agg">SMS Aggregation</h2>
                 <li><strong>ConversationId:</strong> unique ID for each conversation</li>
                 <li><strong>MessageDirection:</strong> direction (default: "Inbound" or "Outbound") of each message</li>
                 <li><strong>MessageBody:</strong> the body/content of the message</li>
-                <li><strong>EndpointPhoneNumber:</strong> phone number or any other unique identifier for the recipient</li>
+                <li><strong>ContactPhone:</strong> phone number or any other unique identifier for the recipient</li>
             </ul>
 
             <form method="post" enctype="multipart/form-data">

From f222588b8a3658829d36cb611142e5ad1f299dcd Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 20 May 2021 20:53:49 -0600
Subject: [PATCH 51/52] insert original filename

---
 parser/main.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/parser/main.py b/parser/main.py
index bd2ee59..7951b70 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -138,7 +138,8 @@ def process_job(data):
             email_data = {
                 'result_file': result_file,
                 'email': data['email'] or config.TEST_TARGET_EMAIL,
-                'job_type': job_type
+                'job_type': job_type,
+                'original_file': data['original_file']
             }
             res = email_results.apply_async(args=[email_data], countdown=0)
             cleanup = cleanup_files.apply_async(args=[data], countdown=10)
@@ -164,11 +165,11 @@ def email_results(data):
         results = ''.join(['<p>{}results/{}</p>'.format(config.BASE_URL, output_file) for output_file in output])
         msg.html = (
             "<p>Thank you for using the votetripling.org SMS transcript processing tool.</p>"
-            "<p>The {} script successfully processed your data file.</p>"
+            "<p>The {} script successfully processed your data file {}.</p>"
             "<p>Link(s) to download the results:</p>"
             "{}"
             "<p>Your result files will be available for download for {} hours.</p>"
-        ).format(UPLOAD_TYPES[data['job_type']]['name'], results, config.FILE_LIFE)
+        ).format(UPLOAD_TYPES[data['job_type']]['name'], data['original_file'], results, config.FILE_LIFE)
         mail.send(msg)
     return True
 
@@ -243,6 +244,7 @@ def index():
                    'in a few minutes for results.').format(
                        file.filename, UPLOAD_TYPES[upload_type]['name'])
             data = {
+                'original_file': file.filename,
                 'input_file': file_path,
                 'upload_type': upload_type,
                 'email': email,
@@ -290,6 +292,7 @@ def email():
             'upload_type': 'vec_file',
             'result_file': '123.csv|456.csv',
             'email': current_app.config['TEST_TARGET_EMAIL']
+            'original_file': 'EMAIL TEST'
         }
         res = email_results.apply_async(args=[data], countdown=0)
         if res.failed():

From 3b50cacc0c159ba1d47415cd87ab6ec8f75e570c Mon Sep 17 00:00:00 2001
From: Sandra Chung <sandra@moveon.org>
Date: Thu, 20 May 2021 21:36:46 -0600
Subject: [PATCH 52/52] now emailing errors

---
 instance/config.py.example | 10 +++++++
 parser/main.py             | 54 +++++++++++++++++++++++++++-----------
 2 files changed, 48 insertions(+), 16 deletions(-)

diff --git a/instance/config.py.example b/instance/config.py.example
index d01c7b2..8b558d2 100644
--- a/instance/config.py.example
+++ b/instance/config.py.example
@@ -9,6 +9,16 @@ UPLOAD_FOLDER = os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annota
 RESULTS_FOLDER = os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Output_Data'),
 SCRIPTS_FOLDER = os.path.join(os.path.dirname(APP_ROOT), 'Projects/NLP/SMS_Annotation/Code'),
 
+# If PROCESS_ASYNC is set to True, we run scripts in the background and email
+# results with celery, redis and flask-mail.
+# If PROCESS_ASYNC is set to False, we run scripts synchronously and await a
+# link to the results. If a script take too long to run (as they do with larger
+# files), the web app may time out before the script finishes.
+PROCESS_ASYNC = True
+
+# Show full script error output in error message to user
+SHOW_SCRIPT_ERRORS = False
+
 CELERY_BROKER_URL = 'redis://localhost:6379/0'
 CELERY_RESULT_BACKEND = 'redis://localhost:6379/1'
 
diff --git a/parser/main.py b/parser/main.py
index 7951b70..9b9fa15 100644
--- a/parser/main.py
+++ b/parser/main.py
@@ -109,7 +109,7 @@ def process_job(data):
             scripts_folder, input_file, scripts_home_dir, output_file, second_output_file,
             third_output_file)
     elif job_type == 'smsagg_file':
-        cmd = 'python {}/aggregate_text_messages.py -d {} -o {}/{} -a "{}" -af "{}" -t "{}" -in "{}"" -out "{}"'.format(
+        cmd = 'python {}/aggregate_text_messages.py -d {} -o {}/{} -a "{}" -af "{}" -t "{}" -in "{}" -out "{}"'.format(
             scripts_folder, input_file, config.RESULTS_FOLDER,
             output_file, data['aff_regex'], data['aff_regex_final'],
             data['init_triple_phrase'], data['inbound_text'], data['outbound_text'])
@@ -124,8 +124,8 @@ def process_job(data):
     if job_run.returncode: # exit code of 0 is success, 1 is generic error
         status = 'error'
         err_log = job_run.stdout.decode()
-        print("ERROR: Could not process job {} file {} type {}".format(
-            job_id, input_file, job_type))
+        print("ERROR: Could not process file {} as type {}".format(
+            input_file, job_type))
         print(err_log)
     else:
         print("SUCCESS")
@@ -136,17 +136,28 @@ def process_job(data):
                 None, [output_file, second_output_file, third_output_file]))
             result_file = '|'.join([file for file in result_files])
             email_data = {
+                'success': True,
                 'result_file': result_file,
                 'email': data['email'] or config.TEST_TARGET_EMAIL,
                 'job_type': job_type,
-                'original_file': data['original_file']
+                'original_file': data['original_file'],
             }
             res = email_results.apply_async(args=[email_data], countdown=0)
             cleanup = cleanup_files.apply_async(args=[data], countdown=10)
         # TODO: delete input file. If processing wasn't successful, leave the input file
         # for troubleshooting
         return True, output_file, second_output_file, third_output_file
-    else:
+    else: # failed to process
+        if config.PROCESS_ASYNC:
+            email_data = {
+                'success': False,
+                'email': data['email'] or config.TEST_TARGET_EMAIL,
+                'job_type': job_type,
+                'original_file': data['original_file'],
+                'error_log': err_log
+            }
+            res = email_results.apply_async(args=[email_data], countdown=0)
+            cleanup = cleanup_files.apply_async(args=[data], countdown=10)
         return False, err_log, None, None
 
 
@@ -161,15 +172,26 @@ def email_results(data):
                 sender=config.EMAIL_SENDER,
                 recipients=[data['email']])
         # accommodating more than one output file
-        output = data['result_file'].split('|')
-        results = ''.join(['<p>{}results/{}</p>'.format(config.BASE_URL, output_file) for output_file in output])
-        msg.html = (
-            "<p>Thank you for using the votetripling.org SMS transcript processing tool.</p>"
-            "<p>The {} script successfully processed your data file {}.</p>"
-            "<p>Link(s) to download the results:</p>"
-            "{}"
-            "<p>Your result files will be available for download for {} hours.</p>"
-        ).format(UPLOAD_TYPES[data['job_type']]['name'], data['original_file'], results, config.FILE_LIFE)
+        if data['success']:
+            output = data['result_file'].split('|')
+            results = ''.join(['<p>{}results/{}</p>'.format(config.BASE_URL, output_file) for output_file in output])
+            msg.html = (
+                "<p>Thank you for using the votetripling.org SMS transcript processing tool.</p>"
+                "<p>The {} script successfully processed your data file {}.</p>"
+                "<p>Link(s) to download the results:</p>"
+                "{}"
+                "<p>Your result files will be available for download for {} hours.</p>"
+            ).format(UPLOAD_TYPES[data['job_type']]['name'], data['original_file'], 
+                results, config.FILE_LIFE)
+        else: # send along the error log
+            msg.html = (
+                "<p>Thank you for using the votetripling.org SMS transcript processing tool.</p>"
+                "<p>The {} script failed to process your data file {}.</p>"
+                "<p>It returned the following error:</p>"
+                "<pre>{}</pre>"
+                "<p>Please correct the issues and try again.</p>"
+                ).format(UPLOAD_TYPES[data['job_type']]['name'], data['original_file'],
+                    data['error_log'])
         mail.send(msg)
     return True
 
@@ -269,7 +291,7 @@ def index():
                         'Download results:  {}'.format('  '.join(result_links)))
                 else:
                     outcome_msg = 'Error processing file {}'.format(
-                        output) if os.environ['FLASK_ENV'] == 'development' else 'Error processing file'
+                        output) if current_app.config['SHOW_SCRIPT_ERRORS'] else 'Error processing file'
 
             flash(outcome_msg, 'info')
             return redirect(request.url)
@@ -291,7 +313,7 @@ def email():
             'job_id': 1234,
             'upload_type': 'vec_file',
             'result_file': '123.csv|456.csv',
-            'email': current_app.config['TEST_TARGET_EMAIL']
+            'email': current_app.config['TEST_TARGET_EMAIL'],
             'original_file': 'EMAIL TEST'
         }
         res = email_results.apply_async(args=[data], countdown=0)