diff --git a/bridge/xmpp/xmpp.go b/bridge/xmpp/xmpp.go
index 75b2b598..ed99aac2 100644
--- a/bridge/xmpp/xmpp.go
+++ b/bridge/xmpp/xmpp.go
@@ -24,14 +24,16 @@ type Bxmpp struct {
 	connected bool
 	sync.RWMutex
 
-	avatarMap map[string]string
+	avatarAvailability map[string]bool
+	avatarMap          map[string]string
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
 	return &Bxmpp{
-		Config:    cfg,
-		xmppMap:   make(map[string]string),
-		avatarMap: make(map[string]string),
+		Config:             cfg,
+		xmppMap:            make(map[string]string),
+		avatarAvailability: make(map[string]bool),
+		avatarMap:          make(map[string]string),
 	}
 }
 
@@ -244,10 +246,14 @@ func (b *Bxmpp) handleXMPP() error {
 					event = config.EventTopicChange
 				}
 
-				avatar := getAvatar(b.avatarMap, v.Remote, b.General)
-				if avatar == "" {
+				available, sok := b.avatarAvailability[v.Remote]
+				avatar := ""
+				if !sok {
 					b.Log.Debugf("Requesting avatar data")
+					b.avatarAvailability[v.Remote] = false
 					b.xc.AvatarRequestData(v.Remote)
+				} else if available {
+					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
 				}
 
 				msgID := v.ID
@@ -278,6 +284,8 @@ func (b *Bxmpp) handleXMPP() error {
 			}
 		case xmpp.AvatarData:
 			b.handleDownloadAvatar(v)
+			b.avatarAvailability[v.From] = true
+			b.Log.Debugf("Avatar for %s is now available", v.From)
 		case xmpp.Presence:
 			// Do nothing.
 		}