寫令牌到Java中

Question

我有符號化的代碼，我想結果寫入文件中的.txt這樣我就可以使用該文件刪除所有的標點文件

public static void tokenization() throws FileNotFoundException, IOException{ 

    String msg; 

    int numberOfTotalTokens =0; 
    int numberOfMessages = 0; 

    String data = "E:\\Data\\SMSSpamCollection.txt"; 

    FileInputStream fisData = new FileInputStream(data); 
    BufferedReader readBufferData = new BufferedReader(new InputStreamReader(fisData)); 

    try{ 
     while ((msg =readBufferData.readLine()) != null) { 
      int numberOfTokens = 0; 
      StringTokenizer tokens = new StringTokenizer(msg); 
      StringBuilder sb = new StringBuilder(); 
      System.out.println("Before: "+msg); 

      System.out.print("After : "); 
      while (tokens.hasMoreTokens()) { 
       msg = tokens.nextToken(); 
       String msgLower = msg.toLowerCase(); 
       numberOfTokens++; 
       numberOfTotalTokens++; 
      } 

      System.out.println(""); 
      System.out.println("Total Tokens: "+numberOfTokens); 
      System.out.println("\n"); 
      numberOfTokens++; 
      numberOfMessages++; 
     } 
     System.out.println("Total Tokens: "+numberOfTotalTokens); 
     System.out.println("Total Messages: "+numberOfMessages); 
    } 
    catch (Exception e){ 
     System.out.println("Error Exception: "+e.getMessage()); 
    } 
} 

的這個結果代碼設置標記： 例如： 我 名 是 計算器

該令牌集需要寫入到文件中的.txt 我怎麼可以編寫成文件的.txt

Answer 1

我在評論中添加了對代碼的更改。使用此代碼，您可以使用BufferedWriter將每行一個令牌寫入文件output.txt。

一些言論：

• 你的代碼做一些基本的標誌化（documentation），不照顧停止詞像標點符號例如
• StringTokenizer是爲了向後遺留類兼容性，但建議使用正則表達式String.split而不是
• 對於更高級的標記化，您可以使用Apache Lucene庫

希望它有幫助！

public static void tokenization() throws FileNotFoundException, IOException{ 

String msg; 

int numberOfTotalTokens =0; 
int numberOfMessages = 0; 

String data = "E:\\Data\\SMSSpamCollection.txt"; 
// create a new output file output.txt 
String outfilename = "E:\\output.txt"; 
File file =new File(outfilename); 
file.createNewFile(); 

FileInputStream fisData = new FileInputStream(data); 
BufferedReader readBufferData = new BufferedReader(new InputStreamReader(fisData)); 

// create a buffer writer tokDataB 
FileWriter tokData = new FileWriter(outfilename,true); 
BufferedWriter tokDataB = new BufferedWriter(tokData); 

try{ 
    while ((msg =readBufferData.readLine()) != null) { 
     int numberOfTokens = 0; 
     StringTokenizer tokens = new StringTokenizer(msg); 
     StringBuilder sb = new StringBuilder(); 
     System.out.println("Before: "+msg); 

     System.out.print("After : "); 
     while (tokens.hasMoreTokens()) { 
      msg = tokens.nextToken(); 
      String msgLower = msg.toLowerCase(); 

      // write one token per line to output file 
      tokDataB.write(msgLower); 
      tokDataB.write("\n"); 

      numberOfTokens++; 
      numberOfTotalTokens++; 
     } 

     System.out.println(""); 
     System.out.println("Total Tokens: "+numberOfTokens); 
     System.out.println("\n"); 
     numberOfTokens++; 
     numberOfMessages++; 
    } 

    // close output writer 
    tokDataB.close();      

    System.out.println("Total Tokens: "+numberOfTotalTokens); 
    System.out.println("Total Messages: "+numberOfMessages); 
} 
catch (Exception e){ 
    System.out.println("Error Exception: "+e.getMessage()); 
} 

}