Fix expose with bridge and NAT

lib/Ravada/Domain.pm

@@ -3970,7 +3970,7 @@ sub _open_exposed_port($self, $internal_port, $name, $restricted, $remote_ip=und
     $public_port = $self->_set_public_port($id_port, $internal_port, $name, $restricted)
     if !$public_port;
 
-    my $local_ip = $self->_vm->ip;
+    my $local_ip = $self->_vm->interface_ip($remote_ip);
     $sth = $$CONNECTOR->dbh->prepare("UPDATE domain_ports set internal_ip=?"
             ." WHERE id_domain=? AND internal_port=?"
     );
@@ -3998,11 +3998,13 @@ sub _open_exposed_port($self, $internal_port, $name, $restricted, $remote_ip=und
         );
         if ($internal_ip_info->{type} eq 'bridge') {
 
+            my $bridge_ip = ( $self->_vm->bridge_ip($internal_ip) or $local_ip);
+
             my @iptables_arg = ("0.0.0.0/0"
                         ,$internal_ip, 'nat', 'POSTROUTING', 'SNAT',
                         ,{'protocol' => 'tcp'
                             ,'d_port' => $internal_port
-                            ,'to_source' => $local_ip
+                            ,'to_source' => $bridge_ip
                         });
 
             $self->_log_iptable(iptables => \@iptables_arg
@@ -4016,7 +4018,7 @@ sub _open_exposed_port($self, $internal_port, $name, $restricted, $remote_ip=und
                 ,d => $internal_ip
                 ,dport => $internal_port
                 ,j => 'SNAT'
-                ,'to-source' => $local_ip
+                ,'to-source' => $bridge_ip
             );
         }
 
@@ -6793,11 +6795,15 @@ sub _around_ip($orig, $self, @args) {
     if (!$self->readonly() && $self->list_ports()) {
         if ($ip && !$self->_data('ports_exposed')) {
             $self->_data('ports_exposed' => 1);
+            my @ip;
+            my $remote_ip = $self->remote_ip();
+            @ip = ( remote_ip => $remote_ip ) if $remote_ip;
             my $req = Ravada::Request->open_exposed_ports(
                 uid => Ravada::Utils::user_daemon->id
                 ,id_domain => $self->id
                 ,retry => 20
                 ,_force => 1
+                ,@ip
             );
         }
         if (!$ip && $self->_data('ports_exposed')) {

lib/Ravada/VM.pm

@@ -907,6 +907,47 @@ sub _set_display_ip($self, $value) {
     $self->_data( display_ip => $value );
 }
 
+=head2 interface_ip
+
+Returns the Host IP from its network interface
+
+Optionally you can pass a remote IP. Then the proper interface
+will be returned if you connect from a different network than default.
+
+=cut
+
+sub interface_ip($self, $remote_ip=undef) {
+    my $key = '_interface_ip_'.($remote_ip // '');
+    return $self->{$key} if exists $self->{$key};
+
+    my $ip = $self->_interface_ip($remote_ip);
+    $ip = $self->ip() if !defined($ip) || $ip eq '';
+
+    $self->{$key} = $ip;
+    return $ip;
+}
+
+=head2 bridge_ip
+
+Returns the Host IP from its bridged network interface that matches the machine IP
+
+Argument: The virtual machine IP
+
+=cut
+
+sub bridge_ip($self, $domain_ip) {
+    my $key = '_interface_ip_'.($domain_ip // '');
+    return $self->{$key} if exists $self->{$key};
+
+    my ($out,$err)=$self->run_command("ip","r","get",$domain_ip);
+    my ($ip) = $out =~ /src (\d+\.\d+\.\d+\.\d+)/;
+
+    $self->{$key} = $ip;
+    return $ip;
+}
+
+
+
 sub _list_ip_address($self) {
     my @cmd = ("ip","address","show");
     my ($out, $err) = $self->run_command(@cmd);

t/vm/93_ports_bridge.t

@@ -6,7 +6,8 @@ use Data::Dumper;
 use IPC::Run3;
 use JSON::XS;
 use Test::More;
-
+use YAML qw(LoadFile);
+use XML::LibXML;
 use lib 't/lib';
 use Test::Ravada;
 
@@ -17,6 +18,11 @@ use_ok('Ravada');
 
 my $BASE_NAME = "zz-test-base-alpine-q35-uefi";
 my $BASE;
+my $FILE_CONFIG_BRIDGE = "t/etc/bridge.conf";
+
+my $CONFIG_BRIDGE;
+$CONFIG_BRIDGE =  LoadFile($FILE_CONFIG_BRIDGE)
+if -e $FILE_CONFIG_BRIDGE;
 
 #######################################################################
 
@@ -34,13 +40,33 @@ sub _import_base($vm) {
 }
 
 sub _wait_ip($domain) {
-    $domain->start(user => user_admin, remote_ip => '1.2.3.5') unless $domain->is_active();
+    my $remote_ip = '1.2.3.5';
     for ( 1 .. 30 ) {
-        return $domain->ip if $domain->ip;
+        my $ip = $domain->ip();
+        return $ip if $ip;
         diag("Waiting for ".$domain->name. " ip") if !(time % 10);
         sleep 1;
     }
-    confess "Error : no ip for ".$domain->name;
+    confess "Error : no ip for ".$domain->name." Maybe set MAC in YAML file $FILE_CONFIG_BRIDGE";
+}
+
+sub _set_mac_address($domain) {
+    return if ! $CONFIG_BRIDGE || !exists $CONFIG_BRIDGE->{mac};
+    if ($domain->type eq 'KVM') {
+        my $doc = XML::LibXML->load_xml( string => $domain->xml_description());
+        my ($dev) = $doc->findnodes('/domain/devices/interface[@type="bridge"]/mac');
+        $dev->setAttribute('address' => $CONFIG_BRIDGE->{mac});
+        $domain->reload_config($doc);
+    } elsif( $domain->type eq 'Void') {
+        my $ip = $CONFIG_BRIDGE->{ip}
+            or die "Error: missing ip in $FILE_CONFIG_BRIDGE ".Dumper($CONFIG_BRIDGE);
+
+        my $hardware = $domain->_value('hardware');
+        $hardware->{network}->[0]->{address} = $ip;
+        $domain->_store('hardware' => $hardware);
+    } else {
+        die "I don't know how to set mac for ".$domain->type;
+    }
 }
 
 sub _set_bridge($vm, $domain) {
@@ -57,6 +83,7 @@ sub _set_bridge($vm, $domain) {
         }
     );
     wait_request();
+    _set_mac_address($domain);
     return $bridges[0];
 }
 
@@ -155,6 +182,136 @@ sub test_bridge($vm) {
 
 }
 
+sub _get_alternate_ip {
+    my ($in, $out, $err);
+    run3(["ip","route"], \$in, \$out, \$err);
+    die $err if $err;
+    my ($ip) = $out =~ /^\d+\.\d+.* dev virbr.*link src (\d+\.\d+\.\d+\.\d+)/m;
+
+    return $ip if $ip;
+
+    warn "Warning: I can't find an alternate ip from $out. Using localhost" if !$ip;
+
+    return '127.0.0.1';
+
+}
+
+# Test scenario with NAT and Display_ip ########################################
+#
+sub test_bridge_nat($vm) {
+
+    my $nat_ip = '198.18.1.33';
+    my $display_ip = _get_alternate_ip();
+    $vm->nat_ip($nat_ip);
+    $vm->display_ip($display_ip);
+
+    my $domain= $BASE->clone(name => new_domain_name, user => user_admin);
+    is($domain->has_nat_interfaces,1,"Expecting ".$domain->name." has nat "
+        .$vm->name);
+    _set_bridge($vm, $domain);
+    is($domain->has_nat_interfaces,0,"Expecting ".$domain->name." has no nat "
+        .$vm->name) or exit;
+
+    my $internal_port = 22;
+    my $name = "foo";
+    $domain->expose(port => $internal_port, restricted => 1, name => 'ssh');
+
+    my $remote_ip = '10.0.0.1';
+    $domain->start(user => user_admin, remote_ip => $remote_ip);
+    _wait_ip($domain);
+
+    Ravada::Request->start_domain(uid => user_admin->id
+        ,id_domain => $domain->id
+        ,remote_ip => $remote_ip
+    );
+    wait_request(debug => 0 , skip => 'refresh_machine_ports');
+
+    my $internal_ip = _wait_ip($domain);
+    $domain->ip;
+    wait_request(debug => 0);
+
+    my $ip_info = $domain->ip_info();
+    ok($ip_info->{type} eq 'bridge');
+
+    my $internal_net = $internal_ip;
+    $internal_net =~ s{(.*)\.\d+$}{$1.0/24};
+
+    my $local_ip = $vm->ip;
+    my $interface_ip = $vm->interface_ip($remote_ip);
+    my $exposed_port = $domain->exposed_port($internal_port);
+    my $public_port = $exposed_port->{public_port};
+
+    ok($public_port) or die $domain->name;
+
+    isnt($exposed_port->{public_port}, $internal_port) or exit;
+
+    my ($in, $out, $err);
+    run3(['iptables','-t','nat','-L','PREROUTING','-n'],\($in, $out, $err));
+    die $err if $err;
+    my @out = split /\n/,$out;
+    is(grep(/^DNAT.*$interface_ip.*dpt:$public_port to:$internal_ip:$internal_port/,@out),1)
+        or die Dumper(\@out);
+
+    run3(['iptables','-t','nat','-L','POSTROUTING','-n'],\($in, $out, $err));
+    die $err if $err;
+    @out = split /\n/,$out;
+
+    is(grep(/^SNAT.* 0.0.0.0\/0\s+$internal_ip\s+tcp dpt\:$internal_port to\:$interface_ip$/,@out),1);
+
+    run3(['iptables-save','-t','nat'],\($in, $out, $err));
+    die $err if $err;
+    @out = grep /SNAT/, split/\n/,$out;
+    my @snat = grep /SNAT/, @out;
+    is(scalar(@snat),1);
+
+    run3(['iptables','-L','FORWARD','-n'],\($in, $out, $err));
+    die $err if $err;
+    @out = split /\n/,$out;
+    is(grep(m{^ACCEPT.*$internal_net\s+state NEW},@out),1) or die $out;
+
+    run3(['iptables','-L','FORWARD','-n'],\($in, $out, $err));
+    die $err if $err;
+    @out = split /\n/,$out;
+    is(grep(m{^ACCEPT.*$remote_ip\s+$internal_ip.*dpt:$internal_port},@out),1) or die $out;
+    is(grep(m{^DROP.*0.0.0.0.+$internal_ip.*dpt:$internal_port},@out),1) or die $out;
+
+    Ravada::Request->shutdown_domain(
+        uid => user_admin->id
+        ,id_domain => $domain->id
+        ,timeout => 2
+    );
+    wait_request(debug => 0, skip => 'refresh_machine_ports');
+    for ( 1.. 10 ) {
+        run3(['iptables','-t','nat','-L','PREROUTING','-n'],\($in, $out, $err));
+        die $err if $err;
+        @out = split /\n/,$out;
+
+        my ($dnat) = grep(/^DNAT.*$local_ip.*dpt:$public_port to:$internal_ip:$internal_port/,@out);
+        my ($snat) = grep(/^SNAT.* 0.0.0.0\/0\s+$internal_ip\s+tcp dpt\:$internal_port to\:$interface_ip$/,@out);
+        last if !$dnat && !$snat;
+
+        wait_request();
+    }
+
+    run3(['iptables','-t','nat','-L','PREROUTING','-n'],\($in, $out, $err));
+    die $err if $err;
+    @out = split /\n/,$out;
+
+    is(grep(/^DNAT.*$local_ip.*dpt:$public_port to:$internal_ip:$internal_port/,@out),0)
+        or die Dumper(\@out);
+
+    run3(['iptables','-t','nat','-L','POSTROUTING','-n'],\($in, $out, $err));
+    die $err if $err;
+    @out = split /\n/,$out;
+    is(grep(/^SNAT.* 0.0.0.0\/0\s+$internal_ip\s+tcp dpt\:$internal_port to\:$local_ip$/,@out),0)
+        or die Dumper([ grep /^SNAT/, @out]);
+
+    $vm->nat_ip('');
+    $vm->display_ip('');
+    remove_domain($domain);
+}
+
+
 ######################################################################
 
 init();
@@ -179,6 +336,7 @@ for my $vm_name ( reverse vm_names() ) {
 
         flush_rules() if !$<;
         _import_base($vm);
+        test_bridge_nat($vm);
         test_bridge($vm);
     }
 }