Browse Source

First pass at changes needed for module metadata caching

christopher lee 1 year ago
parent
commit
fe1af35107

BIN
db/modules_metadata_base.pstore View File


+ 1
- 0
lib/metasploit/framework/spec/constants.rb View File

@@ -22,6 +22,7 @@ module Metasploit::Framework::Spec::Constants
22 22
     Error
23 23
     External
24 24
     Loader
25
+    Metadata
25 26
     MetasploitClassCompatibilityError
26 27
     Namespace
27 28
     VersionCompatibilityError

+ 0
- 2
lib/msf/core/framework.rb View File

@@ -91,8 +91,6 @@ class Framework
91 91
     events.add_general_subscriber(subscriber)
92 92
     events.add_db_subscriber(subscriber)
93 93
     events.add_ui_subscriber(subscriber)
94
-
95
-    Msf::Modules::Metadata::Cache.instance.init(self)
96 94
   end
97 95
 
98 96
   def inspect

+ 1
- 1
lib/msf/core/module/full_name.rb View File

@@ -21,7 +21,7 @@ module Msf::Module::FullName
21 21
     #
22 22
 
23 23
     def fullname
24
-      type + '/' + (refname.nil? ? "" : refname)
24
+      type + '/' + (refname.nil? ? '' : refname)
25 25
     end
26 26
 
27 27
     def shortname

+ 46
- 53
lib/msf/core/module_manager/cache.rb View File

@@ -98,7 +98,7 @@ module Msf::ModuleManager::Cache
98 98
   end
99 99
 
100 100
   # @overload refresh_cache_from_module_files
101
-  #   Rebuilds database and in-memory cache for all modules.
101
+  #   Rebuilds module metadat store and in-memory cache for all modules.
102 102
   #
103 103
   #   @return [void]
104 104
   # @overload refresh_cache_from_module_files(module_class_or_instance)
@@ -107,14 +107,21 @@ module Msf::ModuleManager::Cache
107 107
   #   @param (see Msf::DBManager#update_module_details)
108 108
   #   @return [void]
109 109
   def refresh_cache_from_module_files(module_class_or_instance = nil)
110
-    if framework_migrated?
111
-      if module_class_or_instance
112
-        framework.db.update_module_details(module_class_or_instance)
113
-      else
114
-        framework.db.update_all_module_details
115
-      end
116
-      refresh_cache_from_database(self.module_paths)
110
+    if module_class_or_instance
111
+      Msf::Modules::Metadata::Cache.instance.refresh_metadata_instance(module_class_or_instance)
112
+    else
113
+      module_sets =
114
+          [
115
+              ['exploit', @framework.exploits],
116
+              ['auxiliary', @framework.auxiliary],
117
+              ['post', @framework.post],
118
+              ['payload', @framework.payloads],
119
+              ['encoder', @framework.encoders],
120
+              ['nop', @framework.nops]
121
+          ]
122
+      Msf::Modules::Metadata::Cache.instance.refresh_metadata(module_sets)
117 123
     end
124
+    refresh_cache_from_database(self.module_paths)
118 125
   end
119 126
 
120 127
   # Refreshes the in-memory cache from the database cache.
@@ -126,19 +133,11 @@ module Msf::ModuleManager::Cache
126 133
 
127 134
   protected
128 135
 
129
-  # Returns whether the framework migrations have been run already.
130
-  #
131
-  # @return [true] if migrations have been run
132
-  # @return [false] otherwise
133
-  def framework_migrated?
134
-    framework.db && framework.db.migrated
135
-  end
136
-
137 136
   # @!attribute [rw] module_info_by_path
138
-  #   @return (see #module_info_by_path_from_database!)
137
+  #   @return (see #module_info_by_path_from_store!)
139 138
   attr_accessor :module_info_by_path
140 139
 
141
-  # Return a module info from Mdm::Module::Details in database.
140
+  # Return a module info from Msf::Modules::Metadata::Obj.
142 141
   #
143 142
   # @note Also sets module_set(module_type)[module_reference_name] to Msf::SymbolicModule if it is not already set.
144 143
   #
@@ -148,41 +147,35 @@ module Msf::ModuleManager::Cache
148 147
   def module_info_by_path_from_database!(allowed_paths=[""])
149 148
     self.module_info_by_path = {}
150 149
 
151
-    if framework_migrated?
152
-      allowed_paths = allowed_paths.map{|x| x + "/"}
153
-
154
-      ActiveRecord::Base.connection_pool.with_connection do
155
-        # TODO record module parent_path in Mdm::Module::Detail so it does not need to be derived from file.
156
-        # Use find_each so Mdm::Module::Details are returned in batches, which will
157
-        # handle the growing number of modules better than all.each.
158
-        Mdm::Module::Detail.find_each do |module_detail|
159
-          path = module_detail.file
160
-          type = module_detail.mtype
161
-          reference_name = module_detail.refname
162
-
163
-          # Skip cached modules that are not in our allowed load paths
164
-          next if allowed_paths.select{|x| path.index(x) == 0}.empty?
165
-
166
-          # The load path is assumed to be the next level above the type directory
167
-          type_dir = File.join('', Mdm::Module::Detail::DIRECTORY_BY_TYPE[type], '')
168
-          parent_path = path.split(type_dir)[0..-2].join(type_dir) # TODO: rewrite
169
-
170
-          module_info_by_path[path] = {
171
-              :reference_name => reference_name,
172
-              :type => type,
173
-              :parent_path => parent_path,
174
-              :modification_time => module_detail.mtime
175
-          }
176
-
177
-          typed_module_set = module_set(type)
178
-
179
-          # Don't want to trigger as {Msf::ModuleSet#create} so check for
180
-          # key instead of using ||= which would call {Msf::ModuleSet#[]}
181
-          # which would potentially call {Msf::ModuleSet#create}.
182
-          unless typed_module_set.has_key? reference_name
183
-            typed_module_set[reference_name] = Msf::SymbolicModule
184
-          end
185
-        end
150
+    allowed_paths = allowed_paths.map{|x| x + "/"}
151
+
152
+    metadata = Msf::Modules::Metadata::Cache.instance.get_metadata
153
+    metadata.each do |module_metadata|
154
+      path = module_metadata.path
155
+      type = module_metadata.type
156
+      reference_name = module_metadata.ref_name
157
+
158
+      # Skip cached modules that are not in our allowed load paths
159
+      next if allowed_paths.select{|x| path.index(x) == 0}.empty?
160
+
161
+      # The load path is assumed to be the next level above the type directory
162
+      type_dir = File.join('', Mdm::Module::Detail::DIRECTORY_BY_TYPE[type], '')
163
+      parent_path = path.split(type_dir)[0..-2].join(type_dir) # TODO: rewrite
164
+
165
+      module_info_by_path[path] = {
166
+          :reference_name => reference_name,
167
+          :type => type,
168
+          :parent_path => parent_path,
169
+          :modification_time => module_metadata.mod_time
170
+      }
171
+
172
+      typed_module_set = module_set(type)
173
+
174
+      # Don't want to trigger as {Msf::ModuleSet#create} so check for
175
+      # key instead of using ||= which would call {Msf::ModuleSet#[]}
176
+      # which would potentially call {Msf::ModuleSet#create}.
177
+      unless typed_module_set.has_key? reference_name
178
+        typed_module_set[reference_name] = Msf::SymbolicModule
186 179
       end
187 180
     end
188 181
 

+ 0
- 2
lib/msf/core/module_set.rb View File

@@ -205,8 +205,6 @@ class Msf::ModuleSet < Hash
205 205
 
206 206
     self[reference_name] = klass
207 207
 
208
-    Msf::Modules::Metadata::Cache.instance.cache_module_metadata(klass)
209
-
210 208
     klass
211 209
   end
212 210
 

+ 5
- 1
lib/msf/core/modules/metadata.rb View File

@@ -1,3 +1,7 @@
1
+# -*- coding: binary -*-
2
+
3
+# Namespace for module metadata related data and operations
1 4
 module Msf::Modules::Metadata
2 5
 
3
-end
6
+end
7
+

+ 64
- 100
lib/msf/core/modules/metadata/cache.rb View File

@@ -2,156 +2,120 @@ require 'singleton'
2 2
 require 'msf/events'
3 3
 require 'rex/ui/text/output/stdio'
4 4
 require 'msf/core/constants'
5
+require 'msf/core/modules/metadata'
5 6
 require 'msf/core/modules/metadata/obj'
6 7
 require 'msf/core/modules/metadata/search'
8
+require 'msf/core/modules/metadata/store'
7 9
 
8 10
 #
9 11
 # Core service class that provides storage of module metadata as well as operations on the metadata.
10 12
 # Note that operations on this metadata are included as separate modules.
11 13
 #
12
-# To prevent excessive startup times module definitions are not parsed for metadata until startup
13
-# is complete. Once startup is complete to prevent CPU spin loading is then done gradually and
14
-# only when an operation using the cache is called is CPU use maximized.
15
-#
16 14
 module Msf
17 15
 module Modules
18 16
 module Metadata
19 17
 
20 18
 class Cache
21 19
   include Singleton
22
-  include ::Msf::UiEventSubscriber
23 20
   include Msf::Modules::Metadata::Search
21
+  include Msf::Modules::Metadata::Store
24 22
 
25 23
   #
26
-  # Init registers this class as a listener to be notified when the console is done loading,
27
-  # this acts a hint to this class to start trickle loading the metadata
24
+  # Refreshes cached module metadata as well as updating the store
28 25
   #
29
-  def init(framework)
30
-    register_ui_listener(framework)
31
-    @framework = framework
26
+  def refresh_metadata_instance(module_instance)
27
+    refresh_metadata_instance_internal(module_instance)
28
+    update_store
32 29
   end
33 30
 
34 31
   #
35
-  # Parses module metadata into an in memory cache.
36
-  #
37
-  # @param klass_or_instance - An instantiated or class instance of a module.
32
+  #  Returns the module data cache, but first ensures all the metadata is loaded
38 33
   #
39
-  def cache_module_metadata(klass_or_instance)
40
-    # Only use cache if db is not in use for now
41
-    return  if @framework.db.active
42
-
43
-    if klass_or_instance.is_a?(Class)
44
-      if @module_load_complete
45
-        add_module_metadata_from_class(klass_or_instance)
46
-      else
47
-        @module_definitions << klass_or_instance
48
-      end
49
-    else
50
-      add_module_metadata_from_instance(klass_or_instance)
51
-    end
34
+  def get_metadata
35
+    wait_for_load
36
+    @module_metadata_cache.values
52 37
   end
53 38
 
54
-  #########
55
-  protected
56
-  #########
57
-
58 39
   #
59
-  # Notify the thread responsible for loading metadata that it can start loading.
40
+  # Checks for modules loaded that are not a part of the cache and updates the underlying store
41
+  # if there are changes.
60 42
   #
61
-  def on_ui_start(rev)
62
-    return if @module_load_complete
63
-    @startup_called = true
43
+  def refresh_metadata(module_sets)
44
+    unchanged_module_references = get_unchanged_module_references
45
+    has_changes = false
46
+    module_sets.each do |mt|
47
+      unchanged_reference_name_set = unchanged_module_references[mt[0]]
48
+
49
+      mt[1].keys.sort.each do |mn|
50
+        next if unchanged_reference_name_set.include? mn
51
+        module_instance = mt[1].create(mn)
52
+        next if not module_instance
53
+        begin
54
+          refresh_metadata_instance_internal(module_instance)
55
+          has_changes = true
56
+        rescue Exception => e
57
+          elog("Error updating module details for #{module_instance.fullname}: #{$!.class} #{$!} : #{e.message}")
58
+        end
59
+      end
60
+    end
61
+
62
+    update_store if has_changes
64 63
   end
65 64
 
66 65
   #
67
-  #  Returns the module data cache, but first ensures all the metadata is loaded
66
+  # Returns  a hash(type->set) which references modules that have not changed.
68 67
   #
69
-  def get_module_metadata_cache
70
-    wait_for_load
71
-    return @module_metadata_cache
68
+  def get_unchanged_module_references
69
+    skip_reference_name_set_by_module_type = Hash.new { |hash, module_type|
70
+      hash[module_type] = Set.new
71
+    }
72
+
73
+    @module_metadata_cache.each_value do |module_metadata|
74
+
75
+      unless module_metadata.path and ::File.exist?(module_metadata.path)
76
+        next
77
+      end
78
+
79
+      if ::File.mtime(module_metadata.path).to_i != module_metadata.mod_time.to_i
80
+        next
81
+      end
82
+
83
+      skip_reference_name_set = skip_reference_name_set_by_module_type[module_metadata.type]
84
+      skip_reference_name_set.add(module_metadata.ref_name)
85
+    end
86
+
87
+    return skip_reference_name_set_by_module_type
72 88
   end
73 89
 
74 90
   #######
75 91
   private
76 92
   #######
77 93
 
78
-  def register_ui_listener(framework)
79
-    begin
80
-      framework.events.add_ui_subscriber(self)
81
-    rescue Exception => e
82
-      elog('Unable to register metadata cache service as UI listener')
83
-    end
84
-  end
85
-
86 94
   def wait_for_load
87
-    if (!@module_load_complete)
88
-      @trickle_load = false
89
-      @console.print_warning('Waiting to finish parsing module metadata')
90
-      @module_load_thread.join
91
-    end
95
+    @load_thread.join unless @store_loaded
92 96
   end
93 97
 
94
-  def add_module_metadata_from_class(module_class_definition)
95
-    begin
96
-      instance = module_class_definition.new
97
-      add_module_metadata_from_instance(instance)
98
-    rescue Exception => e
99
-      elog("Error adding module metadata: #{e.message}")
100
-    end
101
-  end
102
-
103
-  def add_module_metadata_from_instance(module_instance)
104
-    module_metadata = Obj.new(module_instance)
105
-    @module_metadata_cache[get_cache_key(module_instance)] = module_metadata
98
+  def refresh_metadata_instance_internal(module_instance)
99
+    metadata_obj = Obj.new(module_instance)
100
+    @module_metadata_cache[get_cache_key(module_instance)] = metadata_obj
106 101
   end
107 102
 
108 103
   def get_cache_key(module_instance)
109 104
     key = ''
110 105
     key << (module_instance.type.nil? ? '' : module_instance.type)
111 106
     key << '_'
112
-    key << module_instance.name
107
+    key << module_instance.refname
113 108
     return key
114 109
   end
115 110
 
116
-  #
117
-  # This method is used by the @module_load_thread
118
-  #
119
-  def load_module_definitions
120
-    loop do
121
-      if @startup_called
122
-        break;
123
-      end
124
-
125
-      sleep 0.3
126
-    end
127
-
128
-    count = 0
129
-    @module_definitions.each {|module_definition|
130
-      add_module_metadata_from_class(module_definition)
131
-      count = count + 1
132
-      if (@trickle_load && count > @trickle_load_batch)
133
-        sleep @trickle_load_interval
134
-        count = 0
135
-      end
136
-    }
137
-
138
-    @module_load_complete = true
139
-    GC.start(full_mark: true, immediate_sweep: true)
140
-  end
141
-
142 111
   def initialize
143
-    @module_load_complete = false
144
-    @startup_called = false;
145
-    @trickle_load = true
146
-    @trickle_load_batch = 200
147
-    @trickle_load_interval = 0.5
148 112
     @module_metadata_cache = {}
149
-    @module_definitions = []
150
-    @module_load_thread = Thread.new {
151
-      load_module_definitions
152
-    }
153
-
113
+    @store_loaded = false
154 114
     @console = Rex::Ui::Text::Output::Stdio.new
115
+    @load_thread = Thread.new  {
116
+      init_store
117
+      @store_loaded = true
118
+    }
155 119
   end
156 120
 end
157 121
 

+ 24
- 3
lib/msf/core/modules/metadata/obj.rb View File

@@ -1,5 +1,7 @@
1
+require 'msf/core/modules/metadata'
2
+
1 3
 #
2
-# Simple accessor object for storing module metadata.
4
+# Simple object for storing a modules metadata.
3 5
 #
4 6
 module Msf
5 7
 module Modules
@@ -20,14 +22,17 @@ class Obj
20 22
   attr_reader :arch
21 23
   attr_reader :rport
22 24
   attr_reader :targets
25
+  attr_reader :mod_time
26
+  attr_reader :is_install_path
27
+  attr_reader :ref_name
23 28
 
24 29
   def initialize(module_instance)
25 30
     @name = module_instance.name
26 31
     @full_name = module_instance.fullname
27 32
     @disclosure_date = module_instance.disclosure_date
28
-    @rank = module_instance.rank
33
+    @rank = module_instance.rank.to_i
29 34
     @type = module_instance.type
30
-    @description = module_instance.description
35
+    @description = module_instance.description.to_s.strip
31 36
     @author = module_instance.author.map{|x| x.to_s}
32 37
     @references = module_instance.references.map{|x| [x.ctx_id, x.ctx_val].join("-") }
33 38
     @is_server = (module_instance.respond_to?(:stance) and module_instance.stance == "aggressive")
@@ -35,11 +40,27 @@ class Obj
35 40
     @platform = module_instance.platform_to_s
36 41
     @arch = module_instance.arch_to_s
37 42
     @rport = module_instance.datastore['RPORT'].to_s
43
+    @path = module_instance.file_path
44
+    @mod_time = ::File.mtime(@path) rescue Time.now
45
+    @ref_name = module_instance.refname
46
+    install_path = Msf::Config.install_root.to_s
47
+    if (@path.to_s.include? (install_path))
48
+      @path = @path.sub(install_path, '')
49
+      @is_install_path = true
50
+    end
38 51
 
39 52
     if module_instance.respond_to?(:targets) and module_instance.targets
40 53
       @targets = module_instance.targets.map{|x| x.name}
41 54
     end
42 55
   end
56
+
57
+  def path
58
+    if @is_install_path
59
+      return ::File.join(Msf::Config.install_root, @path)
60
+    end
61
+
62
+    @path
63
+  end
43 64
 end
44 65
 end
45 66
 end

+ 5
- 2
lib/msf/core/modules/metadata/search.rb View File

@@ -1,3 +1,5 @@
1
+require 'msf/core/modules/metadata'
2
+
1 3
 #
2 4
 # Provides search operations on the module metadata cache.
3 5
 #
@@ -8,7 +10,7 @@ module Msf::Modules::Metadata::Search
8 10
   def find(search_string)
9 11
     search_results = []
10 12
 
11
-    get_module_metadata_cache.values.each { |module_metadata|
13
+    get_metadata.each { |module_metadata|
12 14
       if is_match(search_string, module_metadata)
13 15
         search_results << module_metadata
14 16
       end
@@ -114,4 +116,5 @@ module Msf::Modules::Metadata::Search
114 116
 
115 117
     true
116 118
   end
117
-end
119
+end
120
+

+ 84
- 0
lib/msf/core/modules/metadata/store.rb View File

@@ -0,0 +1,84 @@
1
+require 'pstore'
2
+require 'msf/core/modules/metadata'
3
+
4
+#
5
+# Handles storage of module metadata on disk. A base metadata file is always included - this was added to ensure a much
6
+# better first time user experience as generating the user based metadata file requires 100+ mb at the time of creating
7
+# this module. Subsequent starts of metasploit will load from a user specific metadata file as users potentially load modules
8
+# from other places.
9
+#
10
+module Msf::Modules::Metadata::Store
11
+
12
+  BaseMetaDataFile = 'modules_metadata_base.pstore'
13
+  UserMetaDataFile = 'modules_metadata.pstore'
14
+
15
+  #
16
+  # Initializes from user store (under ~/.msf4) if it exists. else base file (under $INSTALL_ROOT/db) is copied and loaded.
17
+  #
18
+  def init_store
19
+    load_metadata
20
+  end
21
+
22
+  #
23
+  # Update the module meta cache disk store
24
+  #
25
+  def update_store
26
+    @store.transaction do
27
+      @store[:module_metadata] = @module_metadata_cache
28
+    end
29
+  end
30
+
31
+  #######
32
+  private
33
+  #######
34
+
35
+  def load_metadata
36
+    begin
37
+      retries ||= 0
38
+      configure_user_store
39
+      @store = PStore.new(@path_to_user_metadata)
40
+      @module_metadata_cache = @store.transaction(true) { @store[:module_metadata]}
41
+      validate_data if (!@module_metadata_cache.nil? && @module_metadata_cache.size > 0)
42
+      @module_metadata_cache = {} if @module_metadata_cache.nil?
43
+    rescue
44
+      retries +=1
45
+
46
+      # Try to handle the scenario where the file is corrupted
47
+      if (retries < 2 && ::File.exist?(@path_to_user_metadata))
48
+        FileUtils.remove(@path_to_user_metadata, true)
49
+        retry
50
+      else
51
+        @console.print_warning('Unable to load module metadata')
52
+      end
53
+    end
54
+
55
+  end
56
+
57
+  def validate_data
58
+    size_prior = @module_metadata_cache.size
59
+    @module_metadata_cache.delete_if {|path, module_metadata| !::File.exist?(module_metadata.path)}
60
+    update_store if (size_prior != @module_metadata_cache.size)
61
+  end
62
+
63
+  def configure_user_store
64
+    @path_to_user_metadata =  ::File.join(Msf::Config.config_directory, UserMetaDataFile)
65
+    path_to_base_metadata = ::File.join(Msf::Config.install_root, "db", BaseMetaDataFile)
66
+
67
+    if (!::File.exist?(path_to_base_metadata))
68
+      wlog("Missing base module metadata file: #{path_to_base_metadata}")
69
+    else
70
+      if (!::File.exist?(@path_to_user_metadata))
71
+        FileUtils.cp(path_to_base_metadata, @path_to_user_metadata)
72
+        dlog('Created user based module store')
73
+
74
+       # Update the user based module store if an updated base file is created/pushed
75
+      elsif (::File.mtime(path_to_base_metadata).to_i > ::File.mtime(@path_to_user_metadata).to_i)
76
+        FileUtils.remove(@path_to_user_metadata, true)
77
+        FileUtils.cp(path_to_base_metadata, @path_to_user_metadata)
78
+        dlog('Updated user based module store')
79
+      end
80
+    end
81
+  end
82
+
83
+end
84
+

+ 0
- 2
lib/msf/core/payload_set.rb View File

@@ -197,8 +197,6 @@ class PayloadSet < ModuleSet
197 197
     # our own evil purposes.
198 198
     instance = build_payload(payload_module).new
199 199
 
200
-    Msf::Modules::Metadata::Cache.instance.cache_module_metadata(instance)
201
-
202 200
     # Create an array of information about this payload module
203 201
     pinfo =
204 202
       [

+ 5
- 17
lib/msf/ui/console/command_dispatcher/modules.rb View File

@@ -416,26 +416,14 @@ module Msf
416 416
 
417 417
             # Display the table of matches
418 418
             tbl = generate_module_table("Matching Modules", search_term)
419
-            if (framework.db.active)
420
-              framework.search(match, logger: self).each do |m|
421
-                tbl << [
422
-                  m.fullname,
423
-                  m.disclosure_date.nil? ? "" : m.disclosure_date.strftime("%Y-%m-%d"),
419
+            Msf::Modules::Metadata::Cache.instance.find(match).each do |m|
420
+              tbl << [
421
+                  m.full_name,
422
+                  m.disclosure_date.nil? ? '' : m.disclosure_date.strftime("%Y-%m-%d"),
424 423
                   RankingName[m.rank].to_s,
425 424
                   m.name
426
-                ]
427
-              end
428
-            else
429
-              Msf::Modules::Metadata::Cache.instance.find(match).each do |m|
430
-                tbl << [
431
-                    m.full_name,
432
-                    m.disclosure_date.nil? ? '' : m.disclosure_date.strftime("%Y-%m-%d"),
433
-                    Msf::RankingName[m.rank].to_s,
434
-                    m.name
435
-                ]
436
-              end
425
+              ]
437 426
             end
438
-
439 427
             print_line(tbl.to_s)
440 428
           end
441 429
 

+ 1
- 1
lib/msf/ui/console/driver.rb View File

@@ -219,7 +219,7 @@ class Driver < Msf::Ui::Driver
219 219
       self.framework.init_module_paths(module_paths: opts['ModulePath'])
220 220
     end
221 221
 
222
-    if framework.db.active && !opts['DeferModuleLoads']
222
+    if !opts['DeferModuleLoads']
223 223
       framework.threads.spawn("ModuleCacheRebuild", true) do
224 224
         framework.modules.refresh_cache_from_module_files
225 225
       end

+ 67
- 156
spec/support/shared/examples/msf/module_manager/cache.rb View File

@@ -3,6 +3,11 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
3 3
     parent_pathname.to_path
4 4
   end
5 5
 
6
+  let(:metadata_cache) do
7
+    # Msf::Modules::Metadata::Cache.instance.init(framework)
8
+    Msf::Modules::Metadata::Cache.instance
9
+  end
10
+
6 11
   let(:parent_pathname) do
7 12
     Metasploit::Framework.root.join('modules')
8 13
   end
@@ -221,73 +226,37 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
221 226
   end
222 227
 
223 228
   context '#refresh_cache_from_module_files' do
224
-    before(:example) do
225
-      allow(module_manager).to receive(:framework_migrated?).and_return(framework_migrated?)
226
-    end
227 229
 
228
-    context 'with framework migrated' do
229
-      let(:framework_migrated?) do
230
-        true
230
+    context 'with module argument' do
231
+      def refresh_cache_from_module_files
232
+        module_manager.refresh_cache_from_module_files(module_class_or_instance)
231 233
       end
232 234
 
233
-      context 'with module argument' do
234
-        def refresh_cache_from_module_files
235
-          module_manager.refresh_cache_from_module_files(module_class_or_instance)
236
-        end
237
-
238
-        let(:module_class_or_instance) do
239
-          Class.new(Msf::Module)
240
-        end
241
-
242
-        it 'should update database and then update in-memory cache from the database for the given module_class_or_instance' do
243
-          expect(framework.db).to receive(:update_module_details).with(module_class_or_instance).ordered
244
-          expect(module_manager).to receive(:refresh_cache_from_database).ordered
245
-
246
-          refresh_cache_from_module_files
247
-        end
235
+      let(:module_class_or_instance) do
236
+        Class.new(Msf::Module)
248 237
       end
249 238
 
250
-      context 'without module argument' do
251
-        def refresh_cache_from_module_files
252
-          module_manager.refresh_cache_from_module_files
253
-        end
239
+      it 'should update store and then update in-memory cache from the store for the given module_class_or_instance' do
240
+        expect(metadata_cache).to receive(:refresh_metadata_instance).with(module_class_or_instance).ordered
241
+        expect(module_manager).to receive(:refresh_cache_from_database).ordered
254 242
 
255
-        it 'should update database and then update in-memory cache from the database for all modules' do
256
-          expect(framework.db).to receive(:update_all_module_details).ordered
257
-          expect(module_manager).to receive(:refresh_cache_from_database)
258
-
259
-          refresh_cache_from_module_files
260
-        end
243
+        refresh_cache_from_module_files
261 244
       end
262 245
     end
263 246
 
264
-    context 'without framework migrated' do
247
+    context 'without module argument' do
265 248
       def refresh_cache_from_module_files
266 249
         module_manager.refresh_cache_from_module_files
267 250
       end
268 251
 
269
-      let(:framework_migrated?) do
270
-        false
271
-      end
272
-
273
-      it 'should not call Msf::DBManager#update_module_details' do
274
-        expect(framework.db).not_to receive(:update_module_details)
275
-
276
-        refresh_cache_from_module_files
277
-      end
278
-
279
-      it 'should not call Msf::DBManager#update_all_module_details' do
280
-        expect(framework.db).not_to receive(:update_all_module_details)
281
-
282
-        refresh_cache_from_module_files
283
-      end
284
-
285
-      it 'should not call #refresh_cache_from_database' do
286
-        expect(module_manager).not_to receive(:refresh_cache_from_database)
252
+      it 'should update store and then update in-memory cache from the store for all modules' do
253
+        expect(metadata_cache).to receive(:refresh_metadata).ordered
254
+        expect(module_manager).to receive(:refresh_cache_from_database)
287 255
 
288 256
         refresh_cache_from_module_files
289 257
       end
290 258
     end
259
+
291 260
   end
292 261
 
293 262
   context '#refresh_cache_from_database' do
@@ -302,41 +271,6 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
302 271
     end
303 272
   end
304 273
 
305
-  context '#framework_migrated?' do
306
-    subject(:framework_migrated?) do
307
-      module_manager.send(:framework_migrated?)
308
-    end
309
-
310
-    context 'with framework database' do
311
-      before(:example) do
312
-        expect(framework.db).to receive(:migrated).and_return(migrated)
313
-      end
314
-
315
-      context 'with migrated' do
316
-        let(:migrated) do
317
-          true
318
-        end
319
-
320
-        it { is_expected.to be_truthy }
321
-      end
322
-
323
-      context 'without migrated' do
324
-        let(:migrated) do
325
-          false
326
-        end
327
-
328
-        it { is_expected.to be_falsey }
329
-      end
330
-    end
331
-
332
-    context 'without framework database' do
333
-      before(:example) do
334
-        expect(framework).to receive(:db).and_return(nil)
335
-      end
336
-
337
-      it { is_expected.to be_falsey }
338
-    end
339
-  end
340 274
 
341 275
   context '#module_info_by_path' do
342 276
     it 'should have protected method module_info_by_path' do
@@ -359,101 +293,78 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
359 293
       module_manager.send(:module_info_by_path_from_database!)
360 294
     end
361 295
 
362
-    before(:example) do
363
-      allow(module_manager).to receive(:framework_migrated?).and_return(framework_migrated?)
296
+    it 'should call get metadata' do
297
+      allow(metadata_cache).to receive(:get_metadata).and_return([])
298
+      expect(metadata_cache).to receive(:get_metadata)
299
+
300
+      module_info_by_path_from_database!
364 301
     end
365 302
 
366
-    context 'with framework migrated' do
367
-      let(:framework_migrated?) do
368
-        true
303
+    context 'with database cache' do
304
+      #
305
+      # Let!s (let + before(:each))
306
+      #
307
+
308
+      let!(:mdm_module_detail) do
309
+        FactoryGirl.create(:mdm_module_detail,
310
+                           :file => path,
311
+                           :mtype => type,
312
+                           :mtime => pathname.mtime,
313
+                           :refname => reference_name
314
+        )
369 315
       end
370 316
 
371
-      it 'should use ActiveRecord::Batches#find_each to enumerate Mdm::Module::Details in batches' do
372
-        expect(Mdm::Module::Detail).to receive(:find_each)
373
-
317
+      it 'should create cache entry for path' do
374 318
         module_info_by_path_from_database!
319
+
320
+        expect(module_info_by_path).to have_key(path)
375 321
       end
376 322
 
377
-      context 'with database cache' do
378
-        #
379
-        # Let!s (let + before(:each))
380
-        #
381
-
382
-        let!(:mdm_module_detail) do
383
-          FactoryGirl.create(:mdm_module_detail,
384
-                             :file => path,
385
-                             :mtype => type,
386
-                             :mtime => pathname.mtime,
387
-                             :refname => reference_name
388
-          )
323
+      context 'cache entry' do
324
+        subject(:cache_entry) do
325
+          module_info_by_path[path]
389 326
         end
390 327
 
391
-        it 'should create cache entry for path' do
328
+        before(:example) do
392 329
           module_info_by_path_from_database!
393
-
394
-          expect(module_info_by_path).to have_key(path)
395 330
         end
396 331
 
397
-        context 'cache entry' do
398
-          subject(:cache_entry) do
399
-            module_info_by_path[path]
400
-          end
401
-
402
-          before(:example) do
403
-            module_info_by_path_from_database!
404
-          end
332
+        it { expect(subject[:modification_time]).to be_within(1.second).of(pathname_modification_time) }
333
+        it { expect(subject[:parent_path]).to eq(parent_path) }
334
+        it { expect(subject[:reference_name]).to eq(reference_name) }
335
+        it { expect(subject[:type]).to eq(type) }
336
+      end
405 337
 
406
-          it { expect(subject[:modification_time]).to be_within(1.second).of(pathname_modification_time) }
407
-          it { expect(subject[:parent_path]).to eq(parent_path) }
408
-          it { expect(subject[:reference_name]).to eq(reference_name) }
409
-          it { expect(subject[:type]).to eq(type) }
338
+      context 'typed module set' do
339
+        let(:typed_module_set) do
340
+          module_manager.module_set(type)
410 341
         end
411 342
 
412
-        context 'typed module set' do
413
-          let(:typed_module_set) do
414
-            module_manager.module_set(type)
343
+        context 'with reference_name' do
344
+          before(:example) do
345
+            typed_module_set[reference_name] = double('Msf::Module')
415 346
           end
416 347
 
417
-          context 'with reference_name' do
418
-            before(:example) do
419
-              typed_module_set[reference_name] = double('Msf::Module')
420
-            end
421
-
422
-            it 'should not change reference_name value' do
423
-              expect {
424
-                module_info_by_path_from_database!
425
-              }.to_not change {
426
-                typed_module_set[reference_name]
427
-              }
428
-            end
348
+          it 'should not change reference_name value' do
349
+            expect {
350
+              module_info_by_path_from_database!
351
+            }.to_not change {
352
+              typed_module_set[reference_name]
353
+            }
429 354
           end
355
+        end
430 356
 
431
-          context 'without reference_name' do
432
-            it 'should set reference_name value to Msf::SymbolicModule' do
433
-              module_info_by_path_from_database!
357
+        context 'without reference_name' do
358
+          it 'should set reference_name value to Msf::SymbolicModule' do
359
+            module_info_by_path_from_database!
434 360
 
435
-              # have to use fetch because [] will trigger de-symbolization and
436
-              # instantiation.
437
-              expect(typed_module_set.fetch(reference_name)).to eq Msf::SymbolicModule
438
-            end
361
+            # have to use fetch because [] will trigger de-symbolization and
362
+            # instantiation.
363
+            expect(typed_module_set.fetch(reference_name)).to eq Msf::SymbolicModule
439 364
           end
440 365
         end
441 366
       end
442 367
     end
443 368
 
444
-    context 'without framework migrated' do
445
-      let(:framework_migrated?) do
446
-        false
447
-      end
448
-
449
-      it 'should reset #module_info_by_path' do
450
-        # pre-fill module_info_by_path so change can be detected
451
-        module_manager.send(:module_info_by_path=, double('In-memory Cache'))
452
-
453
-        module_info_by_path_from_database!
454
-
455
-        expect(module_info_by_path).to be_empty
456
-      end
457
-    end
458 369
   end
459 370
 end

Loading…
Cancel
Save