package tfidf;
import java.io.*;
import java.util.*;
import org.wltea.analyzer.lucene.IKAnalyzer;
public class ReadFiles {
 /**
   * @param args
   */
 private static ArrayList<String> FileList = new ArrayList<String>();
 // the list of file
 //get list of file for the directory, including sub-directory of it
 public static List<String> readDirs(String filepath) throws FileNotFoundException, IOException
   {
  try
      {
   File file = new File(filepath);
   if(!file.isDirectory())
         {
    System.out.println("输入的[]");
    System.out.println("filepath:" + file.getAbsolutePath());
   } else
         {
    String[] flist = file.list();
    for (int i = 0; i < flist.length; i++)
            {
     File newfile = new File(filepath + "\\" + flist[i]);
     if(!newfile.isDirectory())
               {
      FileList.add(newfile.getAbsolutePath());
     } else if(newfile.isDirectory()) //if file is a directory, call ReadDirs
     {
      readDirs(filepath + "\\" + flist[i]);
     }
    }
   }
  }
  catch(FileNotFoundException e)
      {
   System.out.println(e.getMessage());
  }
  return FileList;
 }
 //read file
 public static String readFile(String file) throws FileNotFoundException, IOException
   {
  StringBuffer strSb = new StringBuffer();
  //String is constant， StringBuffer can be changed.
  InputStreamReader inStrR = new InputStreamReader(new FileInputStream(file), "gbk");
  //byte streams to character streams
  BufferedReader br = new BufferedReader(inStrR);
  String line = br.readLine();
  while(line != null){
   strSb.append(line).append("\r\n");
   line = br.readLine();
  }
  return strSb.toString();
 }
 //word segmentation
 public static ArrayList<String> cutWords(String file) throws IOException{
  ArrayList<String> words = new ArrayList<String>();
  String text = ReadFiles.readFile(file);
  IKAnalyzer analyzer = new IKAnalyzer();
  words = analyzer.split(text);
  return words;
 }
 //term frequency in a file, times for each word
 public static HashMap<String, Integer> normalTF(ArrayList<String> cutwords){
  HashMap<String, Integer> resTF = new HashMap<String, Integer>();
  for (String word : cutwords){
   if(resTF.get(word) == null){
    resTF.put(word, 1);
    System.out.println(word);
   } else{
    resTF.put(word, resTF.get(word) + 1);
    System.out.println(word.toString());
   }
  }
  return resTF;
 }
 //term frequency in a file, frequency of each word
 public static HashMap<String, float> tf(ArrayList<String> cutwords){
  HashMap<String, float> resTF = new HashMap<String, float>();
  int wordLen = cutwords.size();
  HashMap<String, Integer> intTF = ReadFiles.normalTF(cutwords);
  Iterator iter = intTF.entrySet().iterator();
  //iterator for that get from TF
  while(iter.hasNext()){
   Map.Entry entry = (Map.Entry)iter.next();
   resTF.put(entry.getKey().toString(), float.parsefloat(entry.getValue().toString()) / wordLen);
   System.out.println(entry.getKey().toString() + " = "+ float.parsefloat(entry.getValue().toString()) / wordLen);
  }
  return resTF;
 }
 //tf times for file
 public static HashMap<String, HashMap<String, Integer>> normalTFAllFiles(String dirc) throws IOException{
  HashMap<String, HashMap<String, Integer>> allNormalTF = new HashMap<String, HashMap<String,Integer>>();
  List<String> filelist = ReadFiles.readDirs(dirc);
  for (String file : filelist){
   HashMap<String, Integer> dict = new HashMap<String, Integer>();
   ArrayList<String> cutwords = ReadFiles.cutWords(file);
   //get cut word for one file
   dict = ReadFiles.normalTF(cutwords);
   allNormalTF.put(file, dict);
  }
  return allNormalTF;
 }
 //tf for all file
 public static HashMap<String,HashMap<String, float>> tfAllFiles(String dirc) throws IOException{
  HashMap<String, HashMap<String, float>> allTF = new HashMap<String, HashMap<String, float>>();
  List<String> filelist = ReadFiles.readDirs(dirc);
  for (String file : filelist){
   HashMap<String, float> dict = new HashMap<String, float>();
   ArrayList<String> cutwords = ReadFiles.cutWords(file);
   //get cut words for one file
   dict = ReadFiles.tf(cutwords);
   allTF.put(file, dict);
  }
  return allTF;
 }
 public static HashMap<String, float> idf(HashMap<String,HashMap<String, float>> all_tf){
  HashMap<String, float> resIdf = new HashMap<String, float>();
  HashMap<String, Integer> dict = new HashMap<String, Integer>();
  int docNum = FileList.size();
  for (int i = 0; i < docNum; i++){
   HashMap<String, float> temp = all_tf.get(FileList.get(i));
   Iterator iter = temp.entrySet().iterator();
   while(iter.hasNext()){
    Map.Entry entry = (Map.Entry)iter.next();
    String word = entry.getKey().toString();
    if(dict.get(word) == null){
     dict.put(word, 1);
    } else {
     dict.put(word, dict.get(word) + 1);
    }
   }
  }
  System.out.println("IDF for every word is:");
  Iterator iter_dict = dict.entrySet().iterator();
  while(iter_dict.hasNext()){
   Map.Entry entry = (Map.Entry)iter_dict.next();
   float value = (float)Math.log(docNum / float.parsefloat(entry.getValue().toString()));
   resIdf.put(entry.getKey().toString(), value);
   System.out.println(entry.getKey().toString() + " = " + value);
  }
  return resIdf;
 }
 public static void tf_idf(HashMap<String,HashMap<String, float>> all_tf,HashMap<String, float> idfs){
  HashMap<String, HashMap<String, float>> resTfIdf = new HashMap<String, HashMap<String, float>>();
  int docNum = FileList.size();
  for (int i = 0; i < docNum; i++){
   String filepath = FileList.get(i);
   HashMap<String, float> tfidf = new HashMap<String, float>();
   HashMap<String, float> temp = all_tf.get(filepath);
   Iterator iter = temp.entrySet().iterator();
   while(iter.hasNext()){
    Map.Entry entry = (Map.Entry)iter.next();
    String word = entry.getKey().toString();
    float value = (float)float.parsefloat(entry.getValue().toString()) * idfs.get(word);
    tfidf.put(word, value);
   }
   resTfIdf.put(filepath, tfidf);
  }
  System.out.println("TF-IDF for Every file is :");
  DisTfIdf(resTfIdf);
 }
 public static void DisTfIdf(HashMap<String, HashMap<String, float>> tfidf){
  Iterator iter1 = tfidf.entrySet().iterator();
  while(iter1.hasNext()){
   Map.Entry entrys = (Map.Entry)iter1.next();
   System.out.println("FileName: " + entrys.getKey().toString());
   System.out.print("{");
   HashMap<String, float> temp = (HashMap<String, float>) entrys.getValue();
   Iterator iter2 = temp.entrySet().iterator();
   while(iter2.hasNext()){
    Map.Entry entry = (Map.Entry)iter2.next();
    System.out.print(entry.getKey().toString() + " = " + entry.getValue().toString() + ", ");
   }
   System.out.println("}");
  }
 }
 public static void main(String[] args) throws IOException {
  // TODO Auto-generated method stub
  String file = "D:/testfiles";
  HashMap<String,HashMap<String, float>> all_tf = tfAllFiles(file);
  System.out.println();
  HashMap<String, float> idfs = idf(all_tf);
  System.out.println();
  tf_idf(all_tf, idfs);
 }
}