Browse Source

Land #9220, Module cache improvements

Brent Cook 1 year ago
parent
commit
7fe237abe1
No account linked to committer's email address

+ 4
- 0
.gitignore View File

@@ -93,3 +93,7 @@ docker-compose.local*
93 93
 # Ignore python bytecode
94 94
 *.pyc
95 95
 rspec.failures
96
+
97
+
98
+#Ignore any base disk store files
99
+db/modules_metadata_base.pstore

BIN
db/modules_metadata_base.pstore View File


+ 1
- 0
lib/metasploit/framework/spec/constants.rb View File

@@ -22,6 +22,7 @@ module Metasploit::Framework::Spec::Constants
22 22
     Error
23 23
     External
24 24
     Loader
25
+    Metadata
25 26
     MetasploitClassCompatibilityError
26 27
     Namespace
27 28
     VersionCompatibilityError

+ 1
- 17
lib/msf/core/framework.rb View File

@@ -233,24 +233,8 @@ class Framework
233 233
     }
234 234
   end
235 235
 
236
+  # TODO: Anything still using this should be ported to use metadata::cache search
236 237
   def search(match, logger: nil)
237
-    # Check if the database is usable
238
-    use_db = true
239
-    if self.db
240
-      if !(self.db.migrated && self.db.modules_cached)
241
-        logger.print_warning("Module database cache not built yet, using slow search") if logger
242
-        use_db = false
243
-      end
244
-    else
245
-      logger.print_warning("Database not connected, using slow search") if logger
246
-      use_db = false
247
-    end
248
-
249
-    # Used the database for search
250
-    if use_db
251
-      return self.db.search_modules(match)
252
-    end
253
-
254 238
     # Do an in-place search
255 239
     matches = []
256 240
     [ self.exploits, self.auxiliary, self.post, self.payloads, self.nops, self.encoders ].each do |mset|

+ 1
- 1
lib/msf/core/module/full_name.rb View File

@@ -21,7 +21,7 @@ module Msf::Module::FullName
21 21
     #
22 22
 
23 23
     def fullname
24
-      type + '/' + refname
24
+      "#{type}/#{refname}"
25 25
     end
26 26
 
27 27
     def promptname

+ 47
- 53
lib/msf/core/module_manager/cache.rb View File

@@ -3,6 +3,7 @@
3 3
 # Gems
4 4
 #
5 5
 require 'active_support/concern'
6
+require 'msf/core/modules/metadata/cache'
6 7
 
7 8
 # Concerns the module cache maintained by the {Msf::ModuleManager}.
8 9
 module Msf::ModuleManager::Cache
@@ -98,7 +99,7 @@ module Msf::ModuleManager::Cache
98 99
   end
99 100
 
100 101
   # @overload refresh_cache_from_module_files
101
-  #   Rebuilds database and in-memory cache for all modules.
102
+  #   Rebuilds module metadata store and in-memory cache for all modules.
102 103
   #
103 104
   #   @return [void]
104 105
   # @overload refresh_cache_from_module_files(module_class_or_instance)
@@ -107,14 +108,21 @@ module Msf::ModuleManager::Cache
107 108
   #   @param (see Msf::DBManager#update_module_details)
108 109
   #   @return [void]
109 110
   def refresh_cache_from_module_files(module_class_or_instance = nil)
110
-    if framework_migrated?
111
-      if module_class_or_instance
112
-        framework.db.update_module_details(module_class_or_instance)
113
-      else
114
-        framework.db.update_all_module_details
115
-      end
116
-      refresh_cache_from_database(self.module_paths)
111
+    if module_class_or_instance
112
+      Msf::Modules::Metadata::Cache.instance.refresh_metadata_instance(module_class_or_instance)
113
+    else
114
+      module_sets =
115
+          [
116
+              ['exploit', @framework.exploits],
117
+              ['auxiliary', @framework.auxiliary],
118
+              ['post', @framework.post],
119
+              ['payload', @framework.payloads],
120
+              ['encoder', @framework.encoders],
121
+              ['nop', @framework.nops]
122
+          ]
123
+      Msf::Modules::Metadata::Cache.instance.refresh_metadata(module_sets)
117 124
     end
125
+    refresh_cache_from_database(self.module_paths)
118 126
   end
119 127
 
120 128
   # Refreshes the in-memory cache from the database cache.
@@ -126,19 +134,11 @@ module Msf::ModuleManager::Cache
126 134
 
127 135
   protected
128 136
 
129
-  # Returns whether the framework migrations have been run already.
130
-  #
131
-  # @return [true] if migrations have been run
132
-  # @return [false] otherwise
133
-  def framework_migrated?
134
-    framework.db && framework.db.migrated
135
-  end
136
-
137 137
   # @!attribute [rw] module_info_by_path
138
-  #   @return (see #module_info_by_path_from_database!)
138
+  #   @return (see #module_info_by_path_from_store!)
139 139
   attr_accessor :module_info_by_path
140 140
 
141
-  # Return a module info from Mdm::Module::Details in database.
141
+  # Return a module info from Msf::Modules::Metadata::Obj.
142 142
   #
143 143
   # @note Also sets module_set(module_type)[module_reference_name] to Msf::SymbolicModule if it is not already set.
144 144
   #
@@ -148,41 +148,35 @@ module Msf::ModuleManager::Cache
148 148
   def module_info_by_path_from_database!(allowed_paths=[""])
149 149
     self.module_info_by_path = {}
150 150
 
151
-    if framework_migrated?
152
-      allowed_paths = allowed_paths.map{|x| x + "/"}
153
-
154
-      ActiveRecord::Base.connection_pool.with_connection do
155
-        # TODO record module parent_path in Mdm::Module::Detail so it does not need to be derived from file.
156
-        # Use find_each so Mdm::Module::Details are returned in batches, which will
157
-        # handle the growing number of modules better than all.each.
158
-        Mdm::Module::Detail.find_each do |module_detail|
159
-          path = module_detail.file
160
-          type = module_detail.mtype
161
-          reference_name = module_detail.refname
162
-
163
-          # Skip cached modules that are not in our allowed load paths
164
-          next if allowed_paths.select{|x| path.index(x) == 0}.empty?
165
-
166
-          # The load path is assumed to be the next level above the type directory
167
-          type_dir = File.join('', Mdm::Module::Detail::DIRECTORY_BY_TYPE[type], '')
168
-          parent_path = path.split(type_dir)[0..-2].join(type_dir) # TODO: rewrite
169
-
170
-          module_info_by_path[path] = {
171
-              :reference_name => reference_name,
172
-              :type => type,
173
-              :parent_path => parent_path,
174
-              :modification_time => module_detail.mtime
175
-          }
176
-
177
-          typed_module_set = module_set(type)
178
-
179
-          # Don't want to trigger as {Msf::ModuleSet#create} so check for
180
-          # key instead of using ||= which would call {Msf::ModuleSet#[]}
181
-          # which would potentially call {Msf::ModuleSet#create}.
182
-          unless typed_module_set.has_key? reference_name
183
-            typed_module_set[reference_name] = Msf::SymbolicModule
184
-          end
185
-        end
151
+    allowed_paths = allowed_paths.map{|x| x + "/"}
152
+
153
+    metadata = Msf::Modules::Metadata::Cache.instance.get_metadata
154
+    metadata.each do |module_metadata|
155
+      path = module_metadata.path
156
+      type = module_metadata.type
157
+      reference_name = module_metadata.ref_name
158
+
159
+      # Skip cached modules that are not in our allowed load paths
160
+      next if allowed_paths.select{|x| path.index(x) == 0}.empty?
161
+
162
+      # The load path is assumed to be the next level above the type directory
163
+      type_dir = File.join('', Mdm::Module::Detail::DIRECTORY_BY_TYPE[type], '')
164
+      parent_path = path.split(type_dir)[0..-2].join(type_dir) # TODO: rewrite
165
+
166
+      module_info_by_path[path] = {
167
+          :reference_name => reference_name,
168
+          :type => type,
169
+          :parent_path => parent_path,
170
+          :modification_time => module_metadata.mod_time
171
+      }
172
+
173
+      typed_module_set = module_set(type)
174
+
175
+      # Don't want to trigger as {Msf::ModuleSet#create} so check for
176
+      # key instead of using ||= which would call {Msf::ModuleSet#[]}
177
+      # which would potentially call {Msf::ModuleSet#create}.
178
+      unless typed_module_set.has_key? reference_name
179
+        typed_module_set[reference_name] = Msf::SymbolicModule
186 180
       end
187 181
     end
188 182
 

+ 8
- 0
lib/msf/core/modules/metadata.rb View File

@@ -0,0 +1,8 @@
1
+# -*- coding: binary -*-
2
+require 'msf/core/modules'
3
+
4
+# Namespace for module metadata related data and operations
5
+module Msf::Modules::Metadata
6
+
7
+end
8
+

+ 124
- 0
lib/msf/core/modules/metadata/cache.rb View File

@@ -0,0 +1,124 @@
1
+require 'singleton'
2
+require 'msf/events'
3
+require 'rex/ui/text/output/stdio'
4
+require 'msf/core/constants'
5
+require 'msf/core/modules/metadata'
6
+require 'msf/core/modules/metadata/obj'
7
+require 'msf/core/modules/metadata/search'
8
+require 'msf/core/modules/metadata/store'
9
+
10
+#
11
+# Core service class that provides storage of module metadata as well as operations on the metadata.
12
+# Note that operations on this metadata are included as separate modules.
13
+#
14
+module Msf
15
+module Modules
16
+module Metadata
17
+
18
+class Cache
19
+  include Singleton
20
+  include Msf::Modules::Metadata::Search
21
+  include Msf::Modules::Metadata::Store
22
+
23
+  #
24
+  # Refreshes cached module metadata as well as updating the store
25
+  #
26
+  def refresh_metadata_instance(module_instance)
27
+    refresh_metadata_instance_internal(module_instance)
28
+    update_store
29
+  end
30
+
31
+  #
32
+  #  Returns the module data cache, but first ensures all the metadata is loaded
33
+  #
34
+  def get_metadata
35
+    wait_for_load
36
+    @module_metadata_cache.values
37
+  end
38
+
39
+  #
40
+  # Checks for modules loaded that are not a part of the cache and updates the underlying store
41
+  # if there are changes.
42
+  #
43
+  def refresh_metadata(module_sets)
44
+    unchanged_module_references = get_unchanged_module_references
45
+    has_changes = false
46
+    module_sets.each do |mt|
47
+      unchanged_reference_name_set = unchanged_module_references[mt[0]]
48
+
49
+      mt[1].keys.sort.each do |mn|
50
+        next if unchanged_reference_name_set.include? mn
51
+        module_instance = mt[1].create(mn)
52
+        next if not module_instance
53
+        begin
54
+          refresh_metadata_instance_internal(module_instance)
55
+          has_changes = true
56
+        rescue Exception => e
57
+          elog("Error updating module details for #{module_instance.fullname}: #{$!.class} #{$!} : #{e.message}")
58
+        end
59
+      end
60
+    end
61
+
62
+    update_store if has_changes
63
+  end
64
+
65
+  #
66
+  # Returns  a hash(type->set) which references modules that have not changed.
67
+  #
68
+  def get_unchanged_module_references
69
+    skip_reference_name_set_by_module_type = Hash.new { |hash, module_type|
70
+      hash[module_type] = Set.new
71
+    }
72
+
73
+    @module_metadata_cache.each_value do |module_metadata|
74
+
75
+      unless module_metadata.path and ::File.exist?(module_metadata.path)
76
+        next
77
+      end
78
+
79
+      if ::File.mtime(module_metadata.path).to_i != module_metadata.mod_time.to_i
80
+        next
81
+      end
82
+
83
+      skip_reference_name_set = skip_reference_name_set_by_module_type[module_metadata.type]
84
+      skip_reference_name_set.add(module_metadata.ref_name)
85
+    end
86
+
87
+    return skip_reference_name_set_by_module_type
88
+  end
89
+
90
+  #######
91
+  private
92
+  #######
93
+
94
+  def wait_for_load
95
+    @load_thread.join unless @store_loaded
96
+  end
97
+
98
+  def refresh_metadata_instance_internal(module_instance)
99
+    metadata_obj = Obj.new(module_instance)
100
+    @module_metadata_cache[get_cache_key(module_instance)] = metadata_obj
101
+  end
102
+
103
+  def get_cache_key(module_instance)
104
+    key = ''
105
+    key << (module_instance.type.nil? ? '' : module_instance.type)
106
+    key << '_'
107
+    key << module_instance.refname
108
+    return key
109
+  end
110
+
111
+  def initialize
112
+    @module_metadata_cache = {}
113
+    @store_loaded = false
114
+    @console = Rex::Ui::Text::Output::Stdio.new
115
+    @load_thread = Thread.new  {
116
+      init_store
117
+      @store_loaded = true
118
+    }
119
+  end
120
+end
121
+
122
+end
123
+end
124
+end

+ 71
- 0
lib/msf/core/modules/metadata/obj.rb View File

@@ -0,0 +1,71 @@
1
+require 'msf/core/modules/metadata'
2
+
3
+#
4
+# Simple object for storing a modules metadata.
5
+#
6
+module Msf
7
+module Modules
8
+module Metadata
9
+
10
+class Obj
11
+  attr_reader :name
12
+  attr_reader :full_name
13
+  attr_reader :rank
14
+  attr_reader :disclosure_date
15
+  attr_reader :type
16
+  attr_reader :author
17
+  attr_reader :description
18
+  attr_reader :references
19
+  attr_reader :is_server
20
+  attr_reader :is_client
21
+  attr_reader :platform
22
+  attr_reader :arch
23
+  attr_reader :rport
24
+  attr_reader :targets
25
+  attr_reader :mod_time
26
+  attr_reader :is_install_path
27
+  attr_reader :ref_name
28
+
29
+  def initialize(module_instance)
30
+    @name = module_instance.name
31
+    @full_name = module_instance.fullname
32
+    @disclosure_date = module_instance.disclosure_date
33
+    @rank = module_instance.rank.to_i
34
+    @type = module_instance.type
35
+    @description = module_instance.description.to_s.strip
36
+    @author = module_instance.author.map{|x| x.to_s}
37
+    @references = module_instance.references.map{|x| [x.ctx_id, x.ctx_val].join("-") }
38
+    @is_server = (module_instance.respond_to?(:stance) and module_instance.stance == "aggressive")
39
+    @is_client = (module_instance.respond_to?(:stance) and module_instance.stance == "passive")
40
+    @platform = module_instance.platform_to_s
41
+    @arch = module_instance.arch_to_s
42
+    @rport = module_instance.datastore['RPORT'].to_s
43
+    @path = module_instance.file_path
44
+    @mod_time = ::File.mtime(@path) rescue Time.now
45
+    @ref_name = module_instance.refname
46
+    install_path = Msf::Config.install_root.to_s
47
+    if (@path.to_s.include? (install_path))
48
+      @path = @path.sub(install_path, '')
49
+      @is_install_path = true
50
+    end
51
+
52
+    if module_instance.respond_to?(:targets) and module_instance.targets
53
+      @targets = module_instance.targets.map{|x| x.name}
54
+    end
55
+  end
56
+
57
+  def update_mod_time(mod_time)
58
+    @mod_time = mod_time
59
+  end
60
+
61
+  def path
62
+    if @is_install_path
63
+      return ::File.join(Msf::Config.install_root, @path)
64
+    end
65
+
66
+    @path
67
+  end
68
+end
69
+end
70
+end
71
+end

+ 120
- 0
lib/msf/core/modules/metadata/search.rb View File

@@ -0,0 +1,120 @@
1
+require 'msf/core/modules/metadata'
2
+
3
+#
4
+# Provides search operations on the module metadata cache.
5
+#
6
+module Msf::Modules::Metadata::Search
7
+  #
8
+  # Searches the module metadata using the passed search string.
9
+  #
10
+  def find(search_string)
11
+    search_results = []
12
+
13
+    get_metadata.each { |module_metadata|
14
+      if is_match(search_string, module_metadata)
15
+        search_results << module_metadata
16
+      end
17
+    }
18
+
19
+    return search_results
20
+  end
21
+
22
+  #######
23
+  private
24
+  #######
25
+
26
+  def is_match(search_string, module_metadata)
27
+    return false if not search_string
28
+
29
+    search_string += ' '
30
+
31
+    # Split search terms by space, but allow quoted strings
32
+    terms = search_string.split(/\"/).collect{|t| t.strip==t ? t : t.split(' ')}.flatten
33
+    terms.delete('')
34
+
35
+    # All terms are either included or excluded
36
+    res = {}
37
+
38
+    terms.each do |t|
39
+      f,v = t.split(":", 2)
40
+      if not v
41
+        v = f
42
+        f = 'text'
43
+      end
44
+      next if v.length == 0
45
+      f.downcase!
46
+      v.downcase!
47
+      res[f] ||=[   [],    []   ]
48
+      if v[0,1] == "-"
49
+        next if v.length == 1
50
+        res[f][1] << v[1,v.length-1]
51
+      else
52
+        res[f][0] << v
53
+      end
54
+    end
55
+
56
+    k = res
57
+
58
+    [0,1].each do |mode|
59
+      match = false
60
+      k.keys.each do |t|
61
+        next if k[t][mode].length == 0
62
+
63
+        k[t][mode].each do |w|
64
+          # Reset the match flag for each keyword for inclusive search
65
+          match = false if mode == 0
66
+
67
+          # Convert into a case-insensitive regex
68
+          r = Regexp.new(Regexp.escape(w), true)
69
+
70
+          case t
71
+            when 'text'
72
+              terms = [module_metadata.name, module_metadata.full_name, module_metadata.description] + module_metadata.references + module_metadata.author
73
+
74
+              if module_metadata.targets
75
+                terms = terms + module_metadata.targets
76
+              end
77
+              match = [t,w] if terms.any? { |x| x =~ r }
78
+            when 'name'
79
+              match = [t,w] if module_metadata.name =~ r
80
+            when 'path'
81
+              match = [t,w] if module_metadata.full_name =~ r
82
+            when 'author'
83
+              match = [t,w] if module_metadata.author.any? { |a| a =~ r }
84
+            when 'os', 'platform'
85
+              match = [t,w] if module_metadata.platform  =~ r or module_metadata.arch  =~ r
86
+
87
+              if module_metadata.targets
88
+                match = [t,w] if module_metadata.targets.any? { |t| t =~ r }
89
+              end
90
+            when 'port'
91
+              match = [t,w] if module_metadata.rport =~ r
92
+            when 'type'
93
+              match = [t,w] if Msf::MODULE_TYPES.any? { |modt| w == modt and module_metadata.type == modt }
94
+            when 'app'
95
+              match = [t,w] if (w == "server" and module_metadata.is_server)
96
+              match = [t,w] if (w == "client" and module_metadata.is_client)
97
+            when 'cve'
98
+              match = [t,w] if module_metadata.references.any? { |ref| ref =~ /^cve\-/i and ref =~ r }
99
+            when 'bid'
100
+              match = [t,w] if module_metadata.references.any? { |ref| ref =~ /^bid\-/i and ref =~ r }
101
+            when 'edb'
102
+              match = [t,w] if module_metadata.references.any? { |ref| ref =~ /^edb\-/i and ref =~ r }
103
+          end
104
+          break if match
105
+        end
106
+        # Filter this module if no matches for a given keyword type
107
+        if mode == 0 and not match
108
+          return false
109
+        end
110
+      end
111
+      # Filter this module if we matched an exclusion keyword (-value)
112
+      if mode == 1 and match
113
+        return false
114
+      end
115
+    end
116
+
117
+    true
118
+  end
119
+end
120
+

+ 112
- 0
lib/msf/core/modules/metadata/store.rb View File

@@ -0,0 +1,112 @@
1
+require 'pstore'
2
+require 'msf/core/modules/metadata'
3
+
4
+#
5
+# Handles storage of module metadata on disk. A base metadata file is always included - this was added to ensure a much
6
+# better first time user experience as generating the user based metadata file requires 100+ mb at the time of creating
7
+# this module. Subsequent starts of metasploit will load from a user specific metadata file as users potentially load modules
8
+# from other places.
9
+#
10
+module Msf::Modules::Metadata::Store
11
+
12
+  BaseMetaDataFile = 'modules_metadata_base.pstore'
13
+  UserMetaDataFile = 'modules_metadata.pstore'
14
+
15
+  #
16
+  # Initializes from user store (under ~/store/.msf4) if it exists. else base file (under $INSTALL_ROOT/db) is copied and loaded.
17
+  #
18
+  def init_store
19
+    load_metadata
20
+  end
21
+
22
+  #
23
+  # Update the module meta cache disk store
24
+  #
25
+  def update_store
26
+    begin
27
+      @store.transaction do
28
+        @store[:module_metadata] = @module_metadata_cache
29
+      end
30
+    rescue Excepion => e
31
+      elog("Unable to update metadata store: #{e.message}")
32
+    end
33
+  end
34
+
35
+  #######
36
+  private
37
+  #######
38
+
39
+  def load_metadata
40
+    begin
41
+      retries ||= 0
42
+      copied = configure_user_store
43
+      @store = PStore.new(@path_to_user_metadata, true)
44
+      @module_metadata_cache = @store.transaction(true) { @store[:module_metadata]}
45
+      validate_data(copied) if (!@module_metadata_cache.nil? && @module_metadata_cache.size > 0)
46
+      @module_metadata_cache = {} if @module_metadata_cache.nil?
47
+    rescue Exception => e
48
+      retries +=1
49
+
50
+      # Try to handle the scenario where the file is corrupted
51
+      if (retries < 2 && ::File.exist?(@path_to_user_metadata))
52
+        elog('Possible corrupt user metadata store, attempting restore')
53
+        FileUtils.remove(@path_to_user_metadata)
54
+        retry
55
+      else
56
+        @console.print_warning('Unable to load module metadata from disk see error log')
57
+        elog("Unable to load module metadata: #{e.message}")
58
+      end
59
+    end
60
+
61
+  end
62
+
63
+  def validate_data(copied)
64
+    size_prior = @module_metadata_cache.size
65
+    @module_metadata_cache.delete_if {|key, module_metadata| !::File.exist?(module_metadata.path)}
66
+
67
+    if (copied)
68
+      @module_metadata_cache.each_value {|module_metadata|
69
+        module_metadata.update_mod_time(::File.mtime(module_metadata.path))
70
+      }
71
+    end
72
+
73
+    update_store if (size_prior != @module_metadata_cache.size || copied)
74
+  end
75
+
76
+  def configure_user_store
77
+    copied = false
78
+    @path_to_user_metadata = get_user_store
79
+    path_to_base_metadata = ::File.join(Msf::Config.install_root, "db", BaseMetaDataFile)
80
+    user_file_exists = ::File.exist?(@path_to_user_metadata)
81
+    base_file_exists = ::File.exist?(path_to_base_metadata)
82
+
83
+    if (!base_file_exists)
84
+      wlog("Missing base module metadata file: #{path_to_base_metadata}")
85
+      return copied if !user_file_exists
86
+    end
87
+
88
+    if (!user_file_exists)
89
+      FileUtils.cp(path_to_base_metadata, @path_to_user_metadata)
90
+      copied = true
91
+
92
+      dlog('Created user based module store')
93
+
94
+     # Update the user based module store if an updated base file is created/pushed
95
+    elsif (::File.mtime(path_to_base_metadata).to_i > ::File.mtime(@path_to_user_metadata).to_i)
96
+      FileUtils.remove(@path_to_user_metadata)
97
+      FileUtils.cp(path_to_base_metadata, @path_to_user_metadata)
98
+      copied = true
99
+      dlog('Updated user based module store')
100
+    end
101
+
102
+    return copied
103
+  end
104
+
105
+  def get_user_store
106
+    store_dir = ::File.join(Msf::Config.config_directory, "store")
107
+    FileUtils.mkdir(store_dir) if !::File.exist?(store_dir)
108
+    return ::File.join(store_dir, UserMetaDataFile)
109
+  end
110
+
111
+end
112
+

+ 5
- 5
lib/msf/ui/console/command_dispatcher/modules.rb View File

@@ -418,12 +418,12 @@ module Msf
418 418
 
419 419
             # Display the table of matches
420 420
             tbl = generate_module_table("Matching Modules", search_term)
421
-            framework.search(match, logger: self).each do |m|
421
+            Msf::Modules::Metadata::Cache.instance.find(match).each do |m|
422 422
               tbl << [
423
-                m.fullname,
424
-                m.disclosure_date.nil? ? "" : m.disclosure_date.strftime("%Y-%m-%d"),
425
-                RankingName[m.rank].to_s,
426
-                m.name
423
+                  m.full_name,
424
+                  m.disclosure_date.nil? ? '' : m.disclosure_date.strftime("%Y-%m-%d"),
425
+                  RankingName[m.rank].to_s,
426
+                  m.name
427 427
               ]
428 428
             end
429 429
             print_line(tbl.to_s)

+ 1
- 1
lib/msf/ui/console/driver.rb View File

@@ -196,7 +196,7 @@ class Driver < Msf::Ui::Driver
196 196
       self.framework.init_module_paths(module_paths: opts['ModulePath'])
197 197
     end
198 198
 
199
-    if framework.db.active && !opts['DeferModuleLoads']
199
+    if !opts['DeferModuleLoads']
200 200
       framework.threads.spawn("ModuleCacheRebuild", true) do
201 201
         framework.modules.refresh_cache_from_module_files
202 202
       end

+ 72
- 156
spec/support/shared/examples/msf/module_manager/cache.rb View File

@@ -1,8 +1,18 @@
1 1
 RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
2
+
3
+  # Wait for data to be loaded
4
+  before(:all) do
5
+    Msf::Modules::Metadata::Cache.instance.get_metadata
6
+  end
7
+
2 8
   let(:parent_path) do
3 9
     parent_pathname.to_path
4 10
   end
5 11
 
12
+  let(:metadata_cache) do
13
+    Msf::Modules::Metadata::Cache.instance
14
+  end
15
+
6 16
   let(:parent_pathname) do
7 17
     Metasploit::Framework.root.join('modules')
8 18
   end
@@ -221,73 +231,37 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
221 231
   end
222 232
 
223 233
   context '#refresh_cache_from_module_files' do
224
-    before(:example) do
225
-      allow(module_manager).to receive(:framework_migrated?).and_return(framework_migrated?)
226
-    end
227 234
 
228
-    context 'with framework migrated' do
229
-      let(:framework_migrated?) do
230
-        true
235
+    context 'with module argument' do
236
+      def refresh_cache_from_module_files
237
+        module_manager.refresh_cache_from_module_files(module_class_or_instance)
231 238
       end
232 239
 
233
-      context 'with module argument' do
234
-        def refresh_cache_from_module_files
235
-          module_manager.refresh_cache_from_module_files(module_class_or_instance)
236
-        end
237
-
238
-        let(:module_class_or_instance) do
239
-          Class.new(Msf::Module)
240
-        end
241
-
242
-        it 'should update database and then update in-memory cache from the database for the given module_class_or_instance' do
243
-          expect(framework.db).to receive(:update_module_details).with(module_class_or_instance).ordered
244
-          expect(module_manager).to receive(:refresh_cache_from_database).ordered
245
-
246
-          refresh_cache_from_module_files
247
-        end
240
+      let(:module_class_or_instance) do
241
+        Class.new(Msf::Module)
248 242
       end
249 243
 
250
-      context 'without module argument' do
251
-        def refresh_cache_from_module_files
252
-          module_manager.refresh_cache_from_module_files
253
-        end
244
+      it 'should update store and then update in-memory cache from the store for the given module_class_or_instance' do
245
+        expect(metadata_cache).to receive(:refresh_metadata_instance).with(module_class_or_instance).ordered
246
+        expect(module_manager).to receive(:refresh_cache_from_database).ordered
254 247
 
255
-        it 'should update database and then update in-memory cache from the database for all modules' do
256
-          expect(framework.db).to receive(:update_all_module_details).ordered
257
-          expect(module_manager).to receive(:refresh_cache_from_database)
258
-
259
-          refresh_cache_from_module_files
260
-        end
248
+        refresh_cache_from_module_files
261 249
       end
262 250
     end
263 251
 
264
-    context 'without framework migrated' do
252
+    context 'without module argument' do
265 253
       def refresh_cache_from_module_files
266 254
         module_manager.refresh_cache_from_module_files
267 255
       end
268 256
 
269
-      let(:framework_migrated?) do
270
-        false
271
-      end
272
-
273
-      it 'should not call Msf::DBManager#update_module_details' do
274
-        expect(framework.db).not_to receive(:update_module_details)
275
-
276
-        refresh_cache_from_module_files
277
-      end
278
-
279
-      it 'should not call Msf::DBManager#update_all_module_details' do
280
-        expect(framework.db).not_to receive(:update_all_module_details)
281
-
282
-        refresh_cache_from_module_files
283
-      end
284
-
285
-      it 'should not call #refresh_cache_from_database' do
286
-        expect(module_manager).not_to receive(:refresh_cache_from_database)
257
+      it 'should update store and then update in-memory cache from the store for all modules' do
258
+        expect(metadata_cache).to receive(:refresh_metadata).ordered
259
+        expect(module_manager).to receive(:refresh_cache_from_database)
287 260
 
288 261
         refresh_cache_from_module_files
289 262
       end
290 263
     end
264
+
291 265
   end
292 266
 
293 267
   context '#refresh_cache_from_database' do
@@ -302,41 +276,6 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
302 276
     end
303 277
   end
304 278
 
305
-  context '#framework_migrated?' do
306
-    subject(:framework_migrated?) do
307
-      module_manager.send(:framework_migrated?)
308
-    end
309
-
310
-    context 'with framework database' do
311
-      before(:example) do
312
-        expect(framework.db).to receive(:migrated).and_return(migrated)
313
-      end
314
-
315
-      context 'with migrated' do
316
-        let(:migrated) do
317
-          true
318
-        end
319
-
320
-        it { is_expected.to be_truthy }
321
-      end
322
-
323
-      context 'without migrated' do
324
-        let(:migrated) do
325
-          false
326
-        end
327
-
328
-        it { is_expected.to be_falsey }
329
-      end
330
-    end
331
-
332
-    context 'without framework database' do
333
-      before(:example) do
334
-        expect(framework).to receive(:db).and_return(nil)
335
-      end
336
-
337
-      it { is_expected.to be_falsey }
338
-    end
339
-  end
340 279
 
341 280
   context '#module_info_by_path' do
342 281
     it 'should have protected method module_info_by_path' do
@@ -359,101 +298,78 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
359 298
       module_manager.send(:module_info_by_path_from_database!)
360 299
     end
361 300
 
362
-    before(:example) do
363
-      allow(module_manager).to receive(:framework_migrated?).and_return(framework_migrated?)
301
+    it 'should call get metadata' do
302
+      allow(metadata_cache).to receive(:get_metadata).and_return([])
303
+      expect(metadata_cache).to receive(:get_metadata)
304
+
305
+      module_info_by_path_from_database!
364 306
     end
365 307
 
366
-    context 'with framework migrated' do
367
-      let(:framework_migrated?) do
368
-        true
308
+    context 'with database cache' do
309
+      #
310
+      # Let!s (let + before(:each))
311
+      #
312
+
313
+      let!(:mdm_module_detail) do
314
+        FactoryGirl.create(:mdm_module_detail,
315
+                           :file => path,
316
+                           :mtype => type,
317
+                           :mtime => pathname.mtime,
318
+                           :refname => reference_name
319
+        )
369 320
       end
370 321
 
371
-      it 'should use ActiveRecord::Batches#find_each to enumerate Mdm::Module::Details in batches' do
372
-        expect(Mdm::Module::Detail).to receive(:find_each)
373
-
322
+      it 'should create cache entry for path' do
374 323
         module_info_by_path_from_database!
324
+
325
+        expect(module_info_by_path).to have_key(path)
375 326
       end
376 327
 
377
-      context 'with database cache' do
378
-        #
379
-        # Let!s (let + before(:each))
380
-        #
381
-
382
-        let!(:mdm_module_detail) do
383
-          FactoryGirl.create(:mdm_module_detail,
384
-                             :file => path,
385
-                             :mtype => type,
386
-                             :mtime => pathname.mtime,
387
-                             :refname => reference_name
388
-          )
328
+      context 'cache entry' do
329
+        subject(:cache_entry) do
330
+          module_info_by_path[path]
389 331
         end
390 332
 
391
-        it 'should create cache entry for path' do
333
+        before(:example) do
392 334
           module_info_by_path_from_database!
393
-
394
-          expect(module_info_by_path).to have_key(path)
395 335
         end
396 336
 
397
-        context 'cache entry' do
398
-          subject(:cache_entry) do
399
-            module_info_by_path[path]
400
-          end
401
-
402
-          before(:example) do
403
-            module_info_by_path_from_database!
404
-          end
337
+        it { expect(subject[:modification_time]).to be_within(1.second).of(pathname_modification_time) }
338
+        it { expect(subject[:parent_path]).to eq(parent_path) }
339
+        it { expect(subject[:reference_name]).to eq(reference_name) }
340
+        it { expect(subject[:type]).to eq(type) }
341
+      end
405 342
 
406
-          it { expect(subject[:modification_time]).to be_within(1.second).of(pathname_modification_time) }
407
-          it { expect(subject[:parent_path]).to eq(parent_path) }
408
-          it { expect(subject[:reference_name]).to eq(reference_name) }
409
-          it { expect(subject[:type]).to eq(type) }
343
+      context 'typed module set' do
344
+        let(:typed_module_set) do
345
+          module_manager.module_set(type)
410 346
         end
411 347
 
412
-        context 'typed module set' do
413
-          let(:typed_module_set) do
414
-            module_manager.module_set(type)
348
+        context 'with reference_name' do
349
+          before(:example) do
350
+            typed_module_set[reference_name] = double('Msf::Module')
415 351
           end
416 352
 
417
-          context 'with reference_name' do
418
-            before(:example) do
419
-              typed_module_set[reference_name] = double('Msf::Module')
420
-            end
421
-
422
-            it 'should not change reference_name value' do
423
-              expect {
424
-                module_info_by_path_from_database!
425
-              }.to_not change {
426
-                typed_module_set[reference_name]
427
-              }
428
-            end
353
+          it 'should not change reference_name value' do
354
+            expect {
355
+              module_info_by_path_from_database!
356
+            }.to_not change {
357
+              typed_module_set[reference_name]
358
+            }
429 359
           end
360
+        end
430 361
 
431
-          context 'without reference_name' do
432
-            it 'should set reference_name value to Msf::SymbolicModule' do
433
-              module_info_by_path_from_database!
362
+        context 'without reference_name' do
363
+          it 'should set reference_name value to Msf::SymbolicModule' do
364
+            module_info_by_path_from_database!
434 365
 
435
-              # have to use fetch because [] will trigger de-symbolization and
436
-              # instantiation.
437
-              expect(typed_module_set.fetch(reference_name)).to eq Msf::SymbolicModule
438
-            end
366
+            # have to use fetch because [] will trigger de-symbolization and
367
+            # instantiation.
368
+            expect(typed_module_set.fetch(reference_name)).to eq Msf::SymbolicModule
439 369
           end
440 370
         end
441 371
       end
442 372
     end
443 373
 
444
-    context 'without framework migrated' do
445
-      let(:framework_migrated?) do
446
-        false
447
-      end
448
-
449
-      it 'should reset #module_info_by_path' do
450
-        # pre-fill module_info_by_path so change can be detected
451
-        module_manager.send(:module_info_by_path=, double('In-memory Cache'))
452
-
453
-        module_info_by_path_from_database!
454
-
455
-        expect(module_info_by_path).to be_empty
456
-      end
457
-    end
458 374
   end
459 375
 end

Loading…
Cancel
Save