main-assoc-rule.cpp

Go to the documentation of this file.

#include "common.hpp"
#include "common/log.h"
#include "common/allocators.hpp"
#include "io/input/transaction_reader/brBufferedTransactionReader.hpp"
#include "io/input/transaction_reader/SortedTransactionReader.hpp" 

#include "io/codec/coder/Coder.hpp"
#include "io/output/assoc_rule_writer/SimpleAssocRuleWriter.hpp"

#include "io/input/transaction_reader/OrderReverser.hpp"
#include "io/db_cache/BuildTreeDBCache.hpp"

#include "util/StreamParser.hpp"
#include "util/FrequentFilter.cpp"
#include "util/Frequent2Filter.cpp"

#include "apriori/bodon/Trie.hpp"
#include "datastructures/trie/edgelist/OrderedEdgelist.hpp"

#include "apriori/bodon/trie/trie_manipulators/FrequentItemInserter.hpp"
#include "apriori/bodon/trie/trie_manipulators/FrequentPairInserter.hpp"
#include "apriori/bodon/trie/trie_manipulators/support_counter/SupportCounterMerge.hpp"
#include "apriori/OneByOneSupportCounter.hpp"
#include "apriori/bodon/inhomogeneous_trie/trie_manipulators/IntersectProPruner.hpp"
#include "apriori/bodon/inhomogeneous_trie/trie_manipulators/CandidateGeneratorPrune.hpp"
#include "apriori/bodon/inhomogeneous_trie/trie_manipulators/InfreqRemover_AssocRuleGen.hpp"

#include "apriori/Apriori.hpp"

#include <vector>
#include <iostream>
#include <string>


std::string file_format;

void init()
{
   file_format = "File format:";
   file_format += "\n\nThe transactionfile is a plan text file. Each row ";
   file_format += "represents a transaction. \n";
   file_format += "A transaction is a set of items seperated by a nonnumeric ";
   file_format += "character.\nIt can be for example a white space, comma, ";
   file_format += "colon, etc.\n";
   file_format += "Items are nonnegative integers.\n";
}
void usage()
{
   std::cerr<<"\nUsage: assoc_rule transactionfile min_supp min_conf min_lift outcomefile [options]\n";
   std::cerr<<"\n transactionfile    file, that contains the tranasctions of items\n";
   std::cerr<<" min_supp\t    absolute support threshold\n";
   std::cerr<<" min_conf\t    confidence threshold\n";
   std::cerr<<" min_lift\t    lift threshold\n";
   std::cerr<<" outcomefile\t    file to write the frequent itemsets and association rules\n\n";
//   std::cerr << file_format;
   std::cerr<<"\t\t\tHave a succesful mining ;-)\n\n";
}

int process_arguments( int argc, char *argv[], counter_t& min_supp, 
                       bool &isrel, double &relminsupp )
{
   if ( argc < 6 )
   {
     log_err(0,"There are 5 mandatory arguments.");
     usage();
     return 2;
   }
   std::string mins=argv[2];
   if (mins[mins.size()-1]=='%') {
     mins.erase(mins.size()-1);
     isrel=true;
     relminsupp=atof(mins.c_str());
     relminsupp/=100;
     log_info(0,"Using relative minimum support of %lg",relminsupp);
     return 0;
   }
   isrel=false;
   int min_supp_i;
   try
   {
      convert(argv[2], min_supp_i);
      if ( min_supp_i <= 0  )
      {
         log_err(0,"%s cannot be converted to a positive integer.",argv[3]);
         return 3;
      }
   }
   catch(BadConversion e)
   {
      log_err(0,"min_supp conversion problem.");
      return 3;
   }
   min_supp = static_cast<counter_t>(min_supp_i);
   log_info(0,"min_supp is set to %d", min_supp);
   return 0;
}

int main( int argc, char *argv[] )
{
   init();
   counter_t min_supp;
   bool relative;
   double relminsupp;
   
   {
      int return_val = 
         process_arguments( argc, argv, min_supp, relative, relminsupp );
      if(return_val)
         return return_val;
   }

   char* input_file = argv[1];
   char* output_file = argv[5];
   float min_conf = atof(argv[3]);
   float min_lift = atof(argv[4]);
   
   try
   {
      // We assume that the transactions does not contain duplicates!!!
      typedef brBufferedTransactionReader< > T_R;
      // Otherwise uncomment this:
      // typedef SortedTransactionReader<LBufferedTransactionReader< >, true> T_R;
      
      T_R::params_t par_i;
      par_i.file_name = input_file;
      par_i.mode=FileReprBase::READ;
      par_i.file_buffer_size = 16 * 1024;
      T_R tr_reader(&par_i);
      std::vector< std::pair<counter_t, item_t> > freq_items_with_counters;
      counter_t nr_of_transactions;
      // The first step of each algorithms is determining the frequent items.
      FrequentFilter<T_R>
         fr_filter(tr_reader);
      log_status(0,"Finding frequent items.");
      fr_filter.findFrequentItems( freq_items_with_counters,  
                                   nr_of_transactions, min_supp);
      
      if(!freq_items_with_counters.empty())
      {
         log_status(0,"Doing decoder.");
         typedef SimpleAssocRuleWriter DF_D;

         DF_D::params_t par_d;
         par_d.file_name = output_file;
         par_d.mode=FileReprBase::WRITE;
//          par_d.numfreq = freq_items_with_counters.size(); // If StatOutput is used!!!
         DF_D df_decoder(&par_d);
         
         typedef Bodon::LeafWithoutConstructor LEAF_WC;  
         typedef Bodon::Leaf LEAF;       
         typedef Bodon::Trie< LEAF, Bodon::OrderedEdgelist<std::vector<Edge> > > TRIE;
         typedef SortedTransactionReader< Coder<T_R, DF_D>, false, false > S_C_T_R;
         const bool ENDONLY = true;
         typedef OrderReverser< bracz::BuildTreeDBCache< 
            S_C_T_R, std::vector<item_t>, bracz::EndPatriciaBuildTree<ENDONLY>, ENDONLY > >S_C;
         S_C::params_t par_c;
         par_c.file_name = input_file;
         par_c.mode=FileReprBase::READ;
         par_c.largest_item = tr_reader.getLargestItem();
         par_c.decoder = &df_decoder;
         par_c.freq_items_with_counters = &freq_items_with_counters;
         par_c.codemode = ASC;
         log_status(0,"Doing sorted codec.");
         S_C sorted_coder(&par_c);

         std::vector< std::pair<counter_t, std::pair<item_t, item_t> > >
            freq_pairs_with_counters;
         Frequent2Filter<S_C> fr_2_filter(
            &sorted_coder );
//   Frequent2FilterOnline<S_C> fr_2_filter(
//      &sorted_coder );
         log_status(0,"Finding frequent pairs.")
            fr_2_filter.findFrequentPairs(freq_pairs_with_counters, min_supp);

         const NEELevel NEE =  NEE_Off;
         const bool DEADENDPRUNE = false;

         typedef bracz::singleualloc<LEAF_WC, 64 * 1024> LEAF_ALLOCATOR;
         typedef Bodon::FrequentItemInserter<DF_D, TRIE, NEE> FII;
         typedef Bodon::SupportCounterMerge<TRIE> SUPP_C_BASE;
         typedef OneByOneSupportCounter<TRIE, S_C, SUPP_C_BASE> SUPP_C;

         typedef Bodon::FrequentPairInserter<DF_D, TRIE, TRIE, LEAF_ALLOCATOR, NEE> FPI;
         typedef Bodon::inhomogeneous_trie::IntersectProPruner<
            DF_D, TRIE, LEAF_WC, LEAF_ALLOCATOR, NEE, DEADENDPRUNE> PRUNER;
         typedef Bodon::inhomogeneous_trie::CandidateGeneratorPrune<PRUNER, DF_D, TRIE, LEAF_ALLOCATOR, NEE, DEADENDPRUNE> CG;
         typedef Bodon::inhomogeneous_trie::InfreqRemover_AssocRuleGen<DF_D, TRIE, LEAF, LEAF_ALLOCATOR, NEE, DEADENDPRUNE> IR;

         typedef Apriori<S_C, DF_D, TRIE, LEAF_ALLOCATOR, FII, FPI, CG, IR, SUPP_C> A;
         TRIE main_trie;
         LEAF_ALLOCATOR s_alloc;
         IR infrequent_remover(main_trie, df_decoder, s_alloc, 
                               min_conf, min_lift, nr_of_transactions);

         FII fii(main_trie, df_decoder);
         A apriori(main_trie, s_alloc, infrequent_remover, sorted_coder, df_decoder, fii);
         log_status(0,"Finding frequent itemsets.")
            apriori.findFrequentItemsets( 
               nr_of_transactions, *par_c.freq_counters,
               freq_pairs_with_counters, min_supp );
      }
   }
   catch (std::ios_base::failure e)
   {
      log_err(0,"Exiting the program due to IO exception");
      return 1;
   }
}

---